def learnModel(data): """ Access the sample information to learn a model. """ print("Learning the model"); # Get the list of training samples samples=os.listdir(data); # Initialize the model model=[]; # Access to each sample for file in samples: if not file.endswith(".zip"): continue; print("\t Processing file " + file) # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); # ############################################### # USE Ground Truth information to learn the model # ############################################### # Get the list of actions for this frame actionList=smp.getActions(); # Iterate for each action in this sample for action in actionList: # Get the action ID, and start and end frames for the action actionID,startFrame,endFrame=action; # NOTE: We use random predictions on this example, therefore, nothing is done with the image. No model is learnt. # Iterate frame by frame to get the information to learn the model #for numFrame in range(startFrame,endFrame): # Get the image for this frame #image=smp.getRGB(numFrame); # ############################################### # Remove the sample object del smp; # Return the model return model;
def predict(model,data,output): """ Access the sample information to predict the pose. """ # Get the list of training samples samples=os.listdir(data); # Access to each sample for file in samples: # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); # Create a random set of actions for this sample numFrame=0; pred=[]; while numFrame<smp.getNumFrames(): # Generate an initial displacement start=numFrame+random.randint(1,100); # Generate the action duration end=min(start+random.randint(10,100),smp.getNumFrames()); # Generate the action ID actionID=random.randint(1,11); # Check if the number of frames are correct if start<end-1 and end<smp.getNumFrames(): # Store the prediction pred.append([actionID,start,end]) # Move ahead numFrame=end+1; # Store the prediction smp.exportPredictions(pred,output); # Remove the sample object del smp;
def predict(model,data,output): """ Access the sample information to predict the pose. """ actionID = 0 #initialize # Get the list of training samples samples=os.listdir(data); print samples # Access to each sample for file in samples: # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); print file # Create a random set of actions for this sample numFrame=0; pred=[]; seqn = os.path.splitext(file)[0]; while numFrame<smp.getNumFrames(): # Generate an initial displacement #start=numFrame+random.randint(1,100); start = numFrame # Generate the action duration #end=min(start+random.randint(10,100),smp.getNumFrames()); end = min(numFrame+30,smp.getNumFrames()) actionFileName = "test_%s_frame%d-%d.avi" % (seqn,start,end) actionFileFullName = "%s%s%s%s%s" % (trainingVideos,os.path.sep,seqn, os.path.sep, actionFileName) w=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)) h=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)) fps=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FPS)) if os.path.exists(actionFileFullName) and os.path.getsize(actionFileFullName) > 0: print actionFileFullName, ' exists' else: out = cv2.VideoWriter(actionFileFullName,cv2.cv.CV_FOURCC('X','V','I','D'),fps,(w,h)) for n in range(start,end): image=smp.getRGB(n+1); #print type(image) #print n #img=cv2.cv.fromarray(image) #cv2.imshow('my',image) #cv2.waitKey(10) #print type(img) out.write(image) out.release() featureFileName = "densetr_%s_frame%d-%d.txt" % (seqn,start,end) featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName) #if not os.path.exists(featureFileFullName): # continue if os.path.exists(featureFileFullName) and os.path.getsize(featureFileFullName) > 0: print featureFileFullName, ' exists' else: fout = open(featureFileFullName,"w") print featureFileFullName cmd = [] cmd.append(denseTrajectoryExe) seqn = os.path.splitext(file)[0]; cmd.append(actionFileFullName) print cmd proc = Popen(cmd, stdout=PIPE, stderr=PIPE) #proc[actionID].stdout.flush() #proc[actionID].stderr.flush() stdout, stderr = proc.communicate() fout.write(stdout) fout.close() hst = numpy.zeros(bowTraj.vocszHOG) if not os.path.exists(featureFileFullName): print featureFileFullName, ' not found' continue htot = 0 fin = open(featureFileFullName,"r") for line in fin: D.read(line) #descHOG.append(D.HOG) #X = bowTraj.calcFeatures(scipy.vstack(tuple(D.HOG))) #actionID = model.predict(X) idx = bowTraj.bowHOG.kmeans.predict(D.HOG) hst[idx] = hst[idx] + 1 htot = htot + 1 if htot > 0: hst = (1.0 / float(htot)) * hst fin.close() print 'Retrieved model:' print model actionID = model.predict(hst)[0] print hst print 'Predicted: ', actionID # Generate the action ID #actionID=random.randint(1,11); # Check if the number of frames are correct if start<end-1 and end<smp.getNumFrames(): # Store the prediction pred.append([actionID,start,end]) # Move ahead #numFrame=end+1; numFrame=start+15; # Store the prediction smp.exportPredictions(pred,output); # Remove the sample object del smp;
def learnModel(data): """ Access the sample information to learn a model. """ print("Learning the model"); # Get the list of training samples samples=os.listdir(data); #samples = ['Seq01.zip', 'Seq03.zip', 'Seq04.zip']; # Hard coded for experiments # Initialize the model #model=[]; model = None yy = [] ff = [] wordIDs = None words = None t1 = time() dataHOG = None fMap = {} fMapS = [] featureVectorNum = 0 fMap = defaultdict(list) print 'Training Set: ', samples if not os.path.exists("Features.mat"): # Access to each sample for file in samples: if not file.endswith(".zip"): continue; print("\t Processing file " + file) # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); # ############################################### # USE Ground Truth information to learn the model # ############################################### # Get the list of actions for this frame actionList=smp.getActions(); # Iterate for each action in this sample proc = {} stdout = {} stderr = {} print actionList sortedActionList = sorted(actionList) noActionList = getNoActionList(sortedActionList) for nal in noActionList: actionList.append([NumActions+1,nal[0],nal[1]]) # continue for action in actionList: # Get the action ID, and start and end frames for the action actionID,startFrame,endFrame=action; print 'Action: ', actionID, '\t', 'startFrame: ', startFrame, '\t', 'endFrame:', endFrame #output = subprocess.check_output('/bin/ps') #print output #print denseTrajectoryExe, os.path.splitext(file)[0], startFrame, endFrame #cmd = [] #cmd.append(denseTrajectoryExe) #seqn = os.path.splitext(file)[0]; #cmd.append('training/train/%s/%s_color.mp4' % (seqn,seqn)) #cmd.append('-S') #cmd.append(str(startFrame)) #cmd.append('-E') #cmd.append(str(endFrame)) #print cmd #proc[actionID] = Popen(cmd, stdout=PIPE, stderr=PIPE) #stdout[actionID], stderr[actionID] = proc[actionID].communicate() #for line in stdout[actionID]: # print line, # NOTE: We use random predictions on this example, therefore, nothing is done with the image. No model is learnt. # Iterate frame by frame to get the information to learn the model seqn = os.path.splitext(file)[0] actionFileName = "output_%s_%d_frame%d-%d.avi" % (seqn,actionID,startFrame,endFrame) actionFileFullName = "%s%s%s%s%s" % (trainingVideos,os.path.sep,seqn, os.path.sep, actionFileName) featureFileName = "densetr_%s_%d_frame%d-%d.txt" % (seqn,actionID,startFrame,endFrame) featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName) #if not os.path.exists(featureFileFullName): # continue if not os.path.exists(actionFileFullName): print actionFileFullName + ' not present' continue if os.path.exists(featureFileFullName) and os.path.getsize(featureFileFullName) > 0: print featureFileFullName, ' exists' elif endFrame - startFrame + 1 <= 15: print featureFileFullName, 'too small' else: fout = open(featureFileFullName,"w") cmd = [] cmd.append(denseTrajectoryExe) seqn = os.path.splitext(file)[0]; cmd.append(actionFileFullName) print cmd proc[actionID] = Popen(cmd, stdout=PIPE, stderr=PIPE) #proc[actionID].stdout.flush() #proc[actionID].stderr.flush() stdout[actionID], stderr[actionID] = proc[actionID].communicate() fout.write(stdout[actionID]) fout.close() #if not os.path.exists('Features.mat'): fin = open(featureFileFullName,"r") for line in fin: D.read(line) #print 'featureVectorNum: ', featureVectorNum fMap[(actionID, startFrame, endFrame)].append(featureVectorNum) descHOG.append(D.HOG) fMapSTuple = (actionID, startFrame, endFrame, featureVectorNum) fMapS.append(fMapSTuple) featureVectorNum = featureVectorNum + 1 #yy.append(actionID) #ff.append(D.frameNum) #break fin.close() #y = numpy.array(yy) #frameNum = numpy.array(ff) #break # TODO: remove #for numFrame in range(startFrame,endFrame): # Get the image for this frame #image=smp.getRGB(numFrame); #img = cv2.cv.fromarray(image) #print type(img) #print actionName #cv2.imshow(actionName,image) #cv2.waitKey(10) # ############################################### # Remove the sample object del smp; #break # TODO: remove if not os.path.exists('Features.mat'): if descHOG: dataHOG = scipy.vstack(tuple(descHOG)) fMapSr = scipy.vstack(tuple(fMapS)) #scipy.io.savemat('Features.mat', {'frameNum':frameNum,'y':y, 'HOG':dataHOG}, format='5') scipy.io.savemat('Features.mat', {'fMap':fMapSr, 'HOG':dataHOG}, format='5') else: dct = {} print 'Loading pre calculated features' scipy.io.loadmat('Features.mat',dct) #y = dct['y'] dataHOG = dct['HOG'] fMapSr = dct['fMap'] for t in fMapSr: fMap[(t[0],t[1],t[2])].append(t[3]) #frameNum = dct['frameNum'] t2 = time() print 'Dense Trajectory Feature Extraction: %f seconds' % (t2-t1) # Extract words if not os.path.exists("BOWFeatures.mat"): bowTraj.build(dataHOG,None,None,None) wordIDs = bowTraj.bowHOG.pred_labels words = bowTraj.bowHOG.centroids #print wordIDs # nearest centroid for word #print words # centroids #t3 = time() #$print 'BoW build : %f seconds' % (t3-t2) #X = bowTraj.calcFeatures(dataHOG,None,None,None) #t4 = time() scipy.io.savemat('BOWFeatures.mat', {'words':words,'wordIDs':wordIDs}, format='5') else: dct2 = {} dct2 = scipy.io.loadmat('BOWFeatures.mat') wordIDs = dct2['wordIDs'] words = dct2['words'] #centroids print 'words.shape', words.shape print 'wordIDs.shape', wordIDs.shape t3 = time() print 'Quantization into words : %f seconds' % (t3-t2) # Now we create feature vectors print 'Creating feature vectors' XX = [] yy = [] print 'Training Set: ', samples for file in samples: if not file.endswith(".zip"): continue; print("\t Processing file " + file) # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); # Get the list of actions for this frame actionList=smp.getActions(); noActionList = getNoActionList(sorted(actionList)) for nal in noActionList: actionList.append([NumActions+1,nal[0],nal[1]]) cnt = 0 for action in actionList: cnt = cnt + 1 # Get the action ID, and start and end frames for the action actionID,startFrame,endFrame=action; print 'PASS 2: ', cnt, ' : ', 'Action: ', actionID, '\t', 'startFrame: ', startFrame, '\t', 'endFrame:', endFrame h = numpy.zeros(bowTraj.vocszHOG) seqn = os.path.splitext(file)[0] featureFileName = "densetr_%s_%d_frame%d-%d.txt" % (seqn,actionID,startFrame,endFrame) featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName) if not os.path.exists(featureFileFullName): print featureFileFullName, ' does not exist' yy.append(actionID) XX.append(h) continue htot = 0 if (actionID,startFrame,endFrame) in fMap: # print (actionID,startFrame,endFrame), fMap[(actionID,startFrame,endFrame)] for fID in fMap[(actionID,startFrame,endFrame)]: idx = wordIDs[fID] h[idx] = h[idx] + 1 htot = htot + 1 if htot > 0: h = (1.0 / float(htot)) * h #print h yy.append(actionID) XX.append(h) X = scipy.vstack(tuple(XX)) y = numpy.array(yy) #print X #print y #X = bowTraj.calcFeatures(dataActionHOG,None,None,None) t4 = time() print 'BoW histogram creation for training samples', (t4-t3) # sys.exit(0) # Create chi squared SVM kernel model clf = SVC(kernel=chi2_kernel) clf.fit(X,y) print clf t5 = time() print 'SVM train : %f seconds', (t5-t4) #numpy.savez('model', X=X, y=y, clf=clf) #scipy.io.savemat('model.mat', {'X':X,'y':y,'clf':clf}, format='5') model = clf; # # Return the model return model;
def load_frame_matrix(file_name, data_dir, drop_no_action=False, down_sample=None): """ Converting everything to grayscale currently Only using frames the corresponding to an action :param file_names: a list of names of files to load (e.g. "Seq01.zip") :return: x, y """ # is this the test or training data # is_training = 'train' in data_dir # read in RGB images from using action sample function actionSample = ActionSample(data_dir + file_name) # read in ground truth seqXX_labels.csv # columns: ActorID, ActionID, StartFrame, EndFrame seq_labels = np.array(actionSample.getActions()) num_frames = actionSample.getNumFrames() # num_frames = seq_labels[:,3].max() # if file_name == 'Seq02.zip': # num_frames = 840 # elif file_name == 'Seq06.zip': # num_frames = 870 # elif file_name == 'Seq08.zip': # num_frames = 960 # elif file_name == 'Seq09.zip': # num_frames = 840 # initialize output data # store each image as a row sample_image = actionSample.getRGB(1) if down_sample is not None: sample_image = im.imresize(sample_image[:, :, 0], size=down_sample) print sample_image.shape x = np.zeros([num_frames, sample_image.shape[0]*sample_image.shape[1]]) y = np.zeros(num_frames) # loop through each frame in the mp4 video for i in range(num_frames): # is this frame part of an action sequence there_is_a_label = np.any(np.logical_and(seq_labels[:, 2] <= i, seq_labels[:, 3] >= i)) if there_is_a_label: # lookup which action this is label_row = np.where((seq_labels[:, 2] <= i) & (seq_labels[:, 3] >= i))[0] num_labels = len(label_row) if num_labels > 1: # multiple action occurring in this frame # choose action that occurs the most most_frames = 0 for l in label_row: if (seq_labels[l, 3] - seq_labels[l, 2]) > most_frames: action = seq_labels[l, 1] most_frames = seq_labels[l, 3] - seq_labels[l, 2] else: action = seq_labels[label_row[0], 1] y[i] = action else: # assign action #12 to the observation y[i] = 12 # load the image and convert it to a gray-scale image matrix img = actionSample.getRGB(i+1) gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) if down_sample is not None: gray_img = im.imresize(gray_img, size=down_sample) x[i, :] = gray_img.flatten() if drop_no_action: keep_rows = np.where(y != 12)[0] x = x[keep_rows, :] y = y[keep_rows] return x, y, seq_labels
from shutil import copyfile from ChalearnLAPEvaluation import evalAction,exportGT_Action from ChalearnLAPSample import ActionSample import cv2 data='./data/'; samples=os.listdir(data); # Initialize the model model=[]; #fourcc = cv2.VideoWriter_fourcc(*'XVID') # Access to each sample for file in samples: # Create the object to access the sample smp=ActionSample(os.path.join(data,file)); # Get the list of actions for this frame actionList=smp.getActions(); print file print smp.getNumFrames() seqn=os.path.splitext(file)[0] name='trainingVideos/'+seqn os.mkdir(name) for action in actionList: # Get the action ID, and start and end frames for the action actionID,startFrame,endFrame=action; print startFrame,endFrame # fourcc=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FOURCC)) w=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)) h=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))