def learnModel(data):
""" Access the sample information to learn a model. """
print("Learning the model");
# Get the list of training samples
samples=os.listdir(data);
# Initialize the model
model=[];
# Access to each sample
for file in samples:
if not file.endswith(".zip"):
continue;
print("\t Processing file " + file)
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
# ###############################################
# USE Ground Truth information to learn the model
# ###############################################
# Get the list of actions for this frame
actionList=smp.getActions();
# Iterate for each action in this sample
for action in actionList:
# Get the action ID, and start and end frames for the action
actionID,startFrame,endFrame=action;
# NOTE: We use random predictions on this example, therefore, nothing is done with the image. No model is learnt.
# Iterate frame by frame to get the information to learn the model
#for numFrame in range(startFrame,endFrame):
# Get the image for this frame
#image=smp.getRGB(numFrame);
# ###############################################
# Remove the sample object
del smp;
# Return the model
return model;
def predict(model,data,output):
""" Access the sample information to predict the pose. """
# Get the list of training samples
samples=os.listdir(data);
# Access to each sample
for file in samples:
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
# Create a random set of actions for this sample
numFrame=0;
pred=[];
while numFrame<smp.getNumFrames():
# Generate an initial displacement
start=numFrame+random.randint(1,100);
# Generate the action duration
end=min(start+random.randint(10,100),smp.getNumFrames());
# Generate the action ID
actionID=random.randint(1,11);
# Check if the number of frames are correct
if start<end-1 and end<smp.getNumFrames():
# Store the prediction
pred.append([actionID,start,end])
# Move ahead
numFrame=end+1;
# Store the prediction
smp.exportPredictions(pred,output);
# Remove the sample object
del smp;
def predict(model,data,output):
""" Access the sample information to predict the pose. """
actionID = 0 #initialize
# Get the list of training samples
samples=os.listdir(data);
print samples
# Access to each sample
for file in samples:
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
print file
# Create a random set of actions for this sample
numFrame=0;
pred=[];
seqn = os.path.splitext(file)[0];
while numFrame<smp.getNumFrames():
# Generate an initial displacement
#start=numFrame+random.randint(1,100);
start = numFrame
# Generate the action duration
#end=min(start+random.randint(10,100),smp.getNumFrames());
end = min(numFrame+30,smp.getNumFrames())
actionFileName = "test_%s_frame%d-%d.avi" % (seqn,start,end)
actionFileFullName = "%s%s%s%s%s" % (trainingVideos,os.path.sep,seqn, os.path.sep, actionFileName)
w=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
h=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
fps=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FPS))
if os.path.exists(actionFileFullName) and os.path.getsize(actionFileFullName) > 0:
print actionFileFullName, ' exists'
else:
out = cv2.VideoWriter(actionFileFullName,cv2.cv.CV_FOURCC('X','V','I','D'),fps,(w,h))
for n in range(start,end):
image=smp.getRGB(n+1);
#print type(image)
#print n
#img=cv2.cv.fromarray(image)
#cv2.imshow('my',image)
#cv2.waitKey(10)
#print type(img)
out.write(image)
out.release()
featureFileName = "densetr_%s_frame%d-%d.txt" % (seqn,start,end)
featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName)
#if not os.path.exists(featureFileFullName):
# continue
if os.path.exists(featureFileFullName) and os.path.getsize(featureFileFullName) > 0:
print featureFileFullName, ' exists'
else:
fout = open(featureFileFullName,"w")
print featureFileFullName
cmd = []
cmd.append(denseTrajectoryExe)
seqn = os.path.splitext(file)[0];
cmd.append(actionFileFullName)
print cmd
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
#proc[actionID].stdout.flush()
#proc[actionID].stderr.flush()
stdout, stderr = proc.communicate()
fout.write(stdout)
fout.close()
hst = numpy.zeros(bowTraj.vocszHOG)
if not os.path.exists(featureFileFullName):
print featureFileFullName, ' not found'
continue
htot = 0
fin = open(featureFileFullName,"r")
for line in fin:
D.read(line)
#descHOG.append(D.HOG)
#X = bowTraj.calcFeatures(scipy.vstack(tuple(D.HOG)))
#actionID = model.predict(X)
idx = bowTraj.bowHOG.kmeans.predict(D.HOG)
hst[idx] = hst[idx] + 1
htot = htot + 1
if htot > 0:
hst = (1.0 / float(htot)) * hst
fin.close()
print 'Retrieved model:'
print model
actionID = model.predict(hst)[0]
print hst
print 'Predicted: ', actionID
# Generate the action ID
#actionID=random.randint(1,11);
# Check if the number of frames are correct
if start<end-1 and end<smp.getNumFrames():
# Store the prediction
pred.append([actionID,start,end])
# Move ahead
#numFrame=end+1;
numFrame=start+15;
# Store the prediction
smp.exportPredictions(pred,output);
# Remove the sample object
del smp;
def learnModel(data):
""" Access the sample information to learn a model. """
print("Learning the model");
# Get the list of training samples
samples=os.listdir(data);
#samples = ['Seq01.zip', 'Seq03.zip', 'Seq04.zip']; # Hard coded for experiments
# Initialize the model
#model=[];
model = None
yy = []
ff = []
wordIDs = None
words = None
t1 = time()
dataHOG = None
fMap = {}
fMapS = []
featureVectorNum = 0
fMap = defaultdict(list)
print 'Training Set: ', samples
if not os.path.exists("Features.mat"):
# Access to each sample
for file in samples:
if not file.endswith(".zip"):
continue;
print("\t Processing file " + file)
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
# ###############################################
# USE Ground Truth information to learn the model
# ###############################################
# Get the list of actions for this frame
actionList=smp.getActions();
# Iterate for each action in this sample
proc = {}
stdout = {}
stderr = {}
print actionList
sortedActionList = sorted(actionList)
noActionList = getNoActionList(sortedActionList)
for nal in noActionList:
actionList.append([NumActions+1,nal[0],nal[1]])
# continue
for action in actionList:
# Get the action ID, and start and end frames for the action
actionID,startFrame,endFrame=action;
print 'Action: ', actionID, '\t', 'startFrame: ', startFrame, '\t', 'endFrame:', endFrame
#output = subprocess.check_output('/bin/ps')
#print output
#print denseTrajectoryExe, os.path.splitext(file)[0], startFrame, endFrame
#cmd = []
#cmd.append(denseTrajectoryExe)
#seqn = os.path.splitext(file)[0];
#cmd.append('training/train/%s/%s_color.mp4' % (seqn,seqn))
#cmd.append('-S')
#cmd.append(str(startFrame))
#cmd.append('-E')
#cmd.append(str(endFrame))
#print cmd
#proc[actionID] = Popen(cmd, stdout=PIPE, stderr=PIPE)
#stdout[actionID], stderr[actionID] = proc[actionID].communicate()
#for line in stdout[actionID]:
# print line,
# NOTE: We use random predictions on this example, therefore, nothing is done with the image. No model is learnt.
# Iterate frame by frame to get the information to learn the model
seqn = os.path.splitext(file)[0]
actionFileName = "output_%s_%d_frame%d-%d.avi" % (seqn,actionID,startFrame,endFrame)
actionFileFullName = "%s%s%s%s%s" % (trainingVideos,os.path.sep,seqn, os.path.sep, actionFileName)
featureFileName = "densetr_%s_%d_frame%d-%d.txt" % (seqn,actionID,startFrame,endFrame)
featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName)
#if not os.path.exists(featureFileFullName):
# continue
if not os.path.exists(actionFileFullName):
print actionFileFullName + ' not present'
continue
if os.path.exists(featureFileFullName) and os.path.getsize(featureFileFullName) > 0:
print featureFileFullName, ' exists'
elif endFrame - startFrame + 1 <= 15:
print featureFileFullName, 'too small'
else:
fout = open(featureFileFullName,"w")
cmd = []
cmd.append(denseTrajectoryExe)
seqn = os.path.splitext(file)[0];
cmd.append(actionFileFullName)
print cmd
proc[actionID] = Popen(cmd, stdout=PIPE, stderr=PIPE)
#proc[actionID].stdout.flush()
#proc[actionID].stderr.flush()
stdout[actionID], stderr[actionID] = proc[actionID].communicate()
fout.write(stdout[actionID])
fout.close()
#if not os.path.exists('Features.mat'):
fin = open(featureFileFullName,"r")
for line in fin:
D.read(line)
#print 'featureVectorNum: ', featureVectorNum
fMap[(actionID, startFrame, endFrame)].append(featureVectorNum)
descHOG.append(D.HOG)
fMapSTuple = (actionID, startFrame, endFrame, featureVectorNum)
fMapS.append(fMapSTuple)
featureVectorNum = featureVectorNum + 1
#yy.append(actionID)
#ff.append(D.frameNum)
#break
fin.close()
#y = numpy.array(yy)
#frameNum = numpy.array(ff)
#break # TODO: remove
#for numFrame in range(startFrame,endFrame):
# Get the image for this frame
#image=smp.getRGB(numFrame);
#img = cv2.cv.fromarray(image)
#print type(img)
#print actionName
#cv2.imshow(actionName,image)
#cv2.waitKey(10)
# ###############################################
# Remove the sample object
del smp;
#break # TODO: remove
if not os.path.exists('Features.mat'):
if descHOG:
dataHOG = scipy.vstack(tuple(descHOG))
fMapSr = scipy.vstack(tuple(fMapS))
#scipy.io.savemat('Features.mat', {'frameNum':frameNum,'y':y, 'HOG':dataHOG}, format='5')
scipy.io.savemat('Features.mat', {'fMap':fMapSr, 'HOG':dataHOG}, format='5')
else:
dct = {}
print 'Loading pre calculated features'
scipy.io.loadmat('Features.mat',dct)
#y = dct['y']
dataHOG = dct['HOG']
fMapSr = dct['fMap']
for t in fMapSr:
fMap[(t[0],t[1],t[2])].append(t[3])
#frameNum = dct['frameNum']
t2 = time()
print 'Dense Trajectory Feature Extraction: %f seconds' % (t2-t1)
# Extract words
if not os.path.exists("BOWFeatures.mat"):
bowTraj.build(dataHOG,None,None,None)
wordIDs = bowTraj.bowHOG.pred_labels
words = bowTraj.bowHOG.centroids
#print wordIDs # nearest centroid for word
#print words # centroids
#t3 = time()
#$print 'BoW build : %f seconds' % (t3-t2)
#X = bowTraj.calcFeatures(dataHOG,None,None,None)
#t4 = time()
scipy.io.savemat('BOWFeatures.mat', {'words':words,'wordIDs':wordIDs}, format='5')
else:
dct2 = {}
dct2 = scipy.io.loadmat('BOWFeatures.mat')
wordIDs = dct2['wordIDs']
words = dct2['words'] #centroids
print 'words.shape', words.shape
print 'wordIDs.shape', wordIDs.shape
t3 = time()
print 'Quantization into words : %f seconds' % (t3-t2)
# Now we create feature vectors
print 'Creating feature vectors'
XX = []
yy = []
print 'Training Set: ', samples
for file in samples:
if not file.endswith(".zip"):
continue;
print("\t Processing file " + file)
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
# Get the list of actions for this frame
actionList=smp.getActions();
noActionList = getNoActionList(sorted(actionList))
for nal in noActionList:
actionList.append([NumActions+1,nal[0],nal[1]])
cnt = 0
for action in actionList:
cnt = cnt + 1
# Get the action ID, and start and end frames for the action
actionID,startFrame,endFrame=action;
print 'PASS 2: ', cnt, ' : ', 'Action: ', actionID, '\t', 'startFrame: ', startFrame, '\t', 'endFrame:', endFrame
h = numpy.zeros(bowTraj.vocszHOG)
seqn = os.path.splitext(file)[0]
featureFileName = "densetr_%s_%d_frame%d-%d.txt" % (seqn,actionID,startFrame,endFrame)
featureFileFullName = "%s%s%s%s%s" % (denseFeatures,os.path.sep,seqn, os.path.sep, featureFileName)
if not os.path.exists(featureFileFullName):
print featureFileFullName, ' does not exist'
yy.append(actionID)
XX.append(h)
continue
htot = 0
if (actionID,startFrame,endFrame) in fMap:
# print (actionID,startFrame,endFrame), fMap[(actionID,startFrame,endFrame)]
for fID in fMap[(actionID,startFrame,endFrame)]:
idx = wordIDs[fID]
h[idx] = h[idx] + 1
htot = htot + 1
if htot > 0:
h = (1.0 / float(htot)) * h
#print h
yy.append(actionID)
XX.append(h)
X = scipy.vstack(tuple(XX))
y = numpy.array(yy)
#print X
#print y
#X = bowTraj.calcFeatures(dataActionHOG,None,None,None)
t4 = time()
print 'BoW histogram creation for training samples', (t4-t3)
# sys.exit(0)
# Create chi squared SVM kernel model
clf = SVC(kernel=chi2_kernel)
clf.fit(X,y)
print clf
t5 = time()
print 'SVM train : %f seconds', (t5-t4)
#numpy.savez('model', X=X, y=y, clf=clf)
#scipy.io.savemat('model.mat', {'X':X,'y':y,'clf':clf}, format='5')
model = clf;
# # Return the model
return model;
def load_frame_matrix(file_name, data_dir, drop_no_action=False, down_sample=None): """ Converting everything to grayscale currently Only using frames the corresponding to an action :param file_names: a list of names of files to load (e.g. "Seq01.zip") :return: x, y """ # is this the test or training data # is_training = 'train' in data_dir # read in RGB images from using action sample function actionSample = ActionSample(data_dir + file_name) # read in ground truth seqXX_labels.csv # columns: ActorID, ActionID, StartFrame, EndFrame seq_labels = np.array(actionSample.getActions()) num_frames = actionSample.getNumFrames() # num_frames = seq_labels[:,3].max() # if file_name == 'Seq02.zip': # num_frames = 840 # elif file_name == 'Seq06.zip': # num_frames = 870 # elif file_name == 'Seq08.zip': # num_frames = 960 # elif file_name == 'Seq09.zip': # num_frames = 840 # initialize output data # store each image as a row sample_image = actionSample.getRGB(1) if down_sample is not None: sample_image = im.imresize(sample_image[:, :, 0], size=down_sample) print sample_image.shape x = np.zeros([num_frames, sample_image.shape[0]*sample_image.shape[1]]) y = np.zeros(num_frames) # loop through each frame in the mp4 video for i in range(num_frames): # is this frame part of an action sequence there_is_a_label = np.any(np.logical_and(seq_labels[:, 2] <= i, seq_labels[:, 3] >= i)) if there_is_a_label: # lookup which action this is label_row = np.where((seq_labels[:, 2] <= i) & (seq_labels[:, 3] >= i))[0] num_labels = len(label_row) if num_labels > 1: # multiple action occurring in this frame # choose action that occurs the most most_frames = 0 for l in label_row: if (seq_labels[l, 3] - seq_labels[l, 2]) > most_frames: action = seq_labels[l, 1] most_frames = seq_labels[l, 3] - seq_labels[l, 2] else: action = seq_labels[label_row[0], 1] y[i] = action else: # assign action #12 to the observation y[i] = 12 # load the image and convert it to a gray-scale image matrix img = actionSample.getRGB(i+1) gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) if down_sample is not None: gray_img = im.imresize(gray_img, size=down_sample) x[i, :] = gray_img.flatten() if drop_no_action: keep_rows = np.where(y != 12)[0] x = x[keep_rows, :] y = y[keep_rows] return x, y, seq_labels
from shutil import copyfile
from ChalearnLAPEvaluation import evalAction,exportGT_Action
from ChalearnLAPSample import ActionSample
import cv2
data='./data/';
samples=os.listdir(data);
# Initialize the model
model=[];
#fourcc = cv2.VideoWriter_fourcc(*'XVID')
# Access to each sample
for file in samples:
# Create the object to access the sample
smp=ActionSample(os.path.join(data,file));
# Get the list of actions for this frame
actionList=smp.getActions();
print file
print smp.getNumFrames()
seqn=os.path.splitext(file)[0]
name='trainingVideos/'+seqn
os.mkdir(name)
for action in actionList:
# Get the action ID, and start and end frames for the action
actionID,startFrame,endFrame=action;
print startFrame,endFrame
# fourcc=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FOURCC))
w=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
h=int(smp.rgb.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
