def pca_computation(bin_dir, in_file, in_pca_dimension, out_file):
verify_files_readable(in_file)
verify_files_writable(out_file)
if in_pca_dimension <= 0:
raise IOError('pca dimension is not a positive integer \n')
run_cmd([join(bin_dir, 'pca_computation'), in_file, out_file,
str(in_pca_dimension)])
def structured_edge_detection(im_file):
bin_dir = os.path.join(config.bin_dir(), 'structured_edge_detection')
output = os.path.join(mkdtemp(), 'output.jpg')
run_cmd([os.path.join(bin_dir, 'edges'), '-i', im_file, '-o',
output, '-m', os.path.join(bin_dir, 'model.yml')])
edges = cv2.imread(output, cv2.CV_LOAD_IMAGE_GRAYSCALE)
return edges
def face_subspace(self):
bin_dir = config.bin_dir()
run_cmd([
os.path.join(bin_dir, 'aff_face_subspace'), self.feature_file,
self.subspace_file, '-n1',
str(self.n1), '-n2',
str(self.n2)
])
def hog_feature_extract(bin_dir, in_file, out_file, hog_feature_type, cell_size,
numDimensions):
""" Compute HOG features """
verify_files_readable(in_file)
verify_files_writable(out_file)
command = [join(bin_dir, 'hog_feature_extract'), in_file, out_file,
str(hog_feature_type), str(cell_size), str(numDimensions)]
run_cmd(command)
def color_surfex_feature_extract(bin_dir, in_file, out_file, color_type, in_intv,
in_nLayer, in_size, in_angle):
""" Compute Color SURFEX """
verify_files_readable(in_file)
verify_files_writable(out_file)
command = [join(bin_dir, 'color_surfex_feature_extract'), in_file, out_file,
str(color_type), str(in_intv), str(in_nLayer), str(in_size),
str(in_angle)]
run_cmd(command)
def vocab_kms(bin_dir, in_file_projection, in_vocab_size, out_file):
verify_files_readable(in_file_projection)
verify_files_writable(out_file)
document = ElementTree.parse(in_file_projection)
data = document.find( 'nDescriptors')
if in_vocab_size > int(data.text):
raise IOError('number of codes is larger than number of descriptors \n')
run_cmd([join(bin_dir, 'vocab_kms'), in_file_projection, out_file,
str(in_vocab_size)])
def hist_kms(bin_dir, in_file_vocab, in_file_projection, out_file):
verify_files_readable([in_file_vocab, in_file_projection])
verify_files_writable(out_file)
run_cmd([
join(bin_dir, 'hist_kms'),
in_file_vocab,
in_file_projection,
out_file,
])
def predict_2step(self, temp_subspace_file, projected_file):
op_labels_file = os.path.join(self.model_dir, 'op_labels.xml')
bin_dir = config.bin_dir()
run_cmd([
os.path.join(bin_dir, 'aff_face_subspace_predict_2step'),
projected_file, temp_subspace_file, op_labels_file,
str(self.nmodes2),
str(self.topn),
str(self.thresh)
])
os.unlink(temp_subspace_file)
return op_labels_file
def sample(self, output_dir):
"""
Sample video loaded in the VideoFFmpegSampler object using ffmpeg.
Args:
output_dir: string with path to folder where we want to store sampled frames.
NOTE!! If output_dir folder already exists:
THE WHOLE FOLDER WILL BE DELETED and the frames extracted in a clean new folder
Returns:
list of strings with each of the sampled frames (full path filenames)
Raises/Assertions:
"""
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
# By specifying -ss (offset) after -i, we sample from the beginning and
# and drop the first frames. This way we can still sample corrupted
# videos where it's not possible to first seek to the offset.
list_of_ffmpeg_params = ['ffmpeg', '-i', self.filename,
'-ss', str(self.offset), '-t',
str(self.duration)]
filter_strings = ''
if self.scale > 1:
filter_strings = 'scale=%d:-1' % (self.scale)
elif self.scale < 1:
filter_strings = 'scale=iw*%f:-1' % (self.scale)
if not self.keyframes:
list_of_ffmpeg_params += ['-r', str(self.fps)]
else:
list_of_ffmpeg_params += ['-vsync', '0']
if len(filter_strings) > 0:
filter_strings = str(filter_strings) + ', '
filter_strings += 'select=eq(pict_type\,I)'
if len(filter_strings) > 0:
list_of_ffmpeg_params += ['-vf', filter_strings]
list_of_ffmpeg_params += ['-f', 'image2',
output_dir + '/%06d.' + self.img_ext]
run_cmd(list_of_ffmpeg_params)
list_of_frames = os.listdir(output_dir)
list_of_frames.sort()
list_of_filenames = \
[os.path.join(output_dir, f)
for f in list_of_frames if f.endswith(self.img_ext)]
return list_of_filenames
def dense_feature_extract(bin_dir, in_file, out_file, in_feature_type, in_intv,
in_nLayer, in_size, in_angle):
""" Compute features using a dense grid """
verify_files_readable(in_file)
verify_files_writable(out_file)
### only feature types: SURFEX(surf extended) and SURF
if in_feature_type == "SURFEX":
run_cmd([join(bin_dir, 'dense_feature_extract'), in_file, out_file,
'SURF', str(in_intv), str(in_nLayer), str(in_size),
str(in_angle), str(1)])
elif in_feature_type == "SURF": ##in_feature_type == "SURF"
run_cmd([join(bin_dir, 'dense_feature_extract'), in_file, out_file,
'SURF', str(in_intv), str(in_nLayer), str(in_size),
str(in_angle), str(0)])
else:
raise IOError('Feature %s not supported' % in_feature_type)
def subspace_project(self, test_features):
projected_file = os.path.join(self.model_dir, 'proj.xml')
# creating copy of subspace file since we over write the same file
temp_subspace_file = os.path.join(
self.model_dir, 'temp_subspace_file_%s.xml' % datetime.utcnow())
shutil.copy(self.subspace_file, temp_subspace_file)
# create a feature file
feature_file = os.path.join(self.model_dir, 'test_features.xml')
la = np.asarray([[-1] * test_features.shape[0]])
cv_feats = np.transpose(np.concatenate(
(la, test_features.transpose())))
mat_to_save = cv_feats.transpose().copy()
cv.Save(feature_file, cv.fromarray(mat_to_save))
# call binary
bin_dir = config.bin_dir()
run_cmd([
os.path.join(bin_dir, 'aff_face_subspace_project'), feature_file,
temp_subspace_file, projected_file
])
return temp_subspace_file, projected_file
def extract_text(self, audio_path, text_path):
"""Run speech model on a audio to generate transcript"""
params = {
'bin_dir': config.bin_dir(),
'raw': audio_path,
'text': text_path,
'hmm': self.hmm,
'lm': self.lm,
'dic': self.dic,
}
try:
run_cmd(TEXT_EXTRACT_CMD, params)
except VideoProcessingError:
# ASR occasionally fails with an error finding the start node or similar
# In that case, save an empty file as transcript for the chunk
# run_cmd already logged the stdout/ stderr from the failed proc
logger.error("Failed running ASR on chunk for video %s",
self.video)
open(text_path, 'w').close()
def generate_transcript(self):
"""Process our video file and write its transcript to self.transcript_path"""
self.grab_s3_files()
clip_length = 10
duration = video_duration(self.video_path)
chunks = int(math.ceil(duration / float(clip_length)))
audio_path = os.path.join(self.base_dir, 'audio.raw')
audio_clip_path = os.path.join(self.base_dir, 'clip.raw')
text_clip_path = os.path.join(self.base_dir, 'clip.txt')
logger.info("generating transcript")
try:
extract_audio(self.video_path, audio_path)
except VideoProcessingError as e:
if 'does not contain any stream' not in str(e):
raise
logger.error("Video %s has no audio stream", self.video)
# Empty transcript because we haven no audio
open(self.transcript_path, 'w').close()
return
for chunk in xrange(chunks):
start = chunk * clip_length
params = {
'infile': audio_path,
'outfile': audio_clip_path,
'secs': start,
'clip_length': clip_length,
'bin_dir': config.bin_dir(),
}
run_cmd(FFMPEG_EXTRACT_AUDIO_CLIP, params)
self.extract_text(audio_clip_path, text_clip_path)
self.append_to_transcript(text_clip_path)
os.unlink(audio_clip_path)
os.unlink(text_clip_path)
logger.info("done generating transcript")
def run_aff_pca(self, features):
'''call aff_face_pca binary for fast PCA computation in C'''
# we need to add an extra id in the start since
#aff_pca_expects it that way
ones = np.zeros((features.shape[0], 1))
temp_feats = np.concatenate((ones, features), axis=1).\
transpose().copy()
temp_dir = mkdtemp()
feature_file = os.path.join(temp_dir, 'features.xml')
cv_mat = cv.fromarray(temp_feats)
cv.Save(feature_file, cv_mat)
pca_file = os.path.join(temp_dir, 'learned_pca.xml')
bin_path = config.bin_dir()
cmd = [
os.path.join(bin_path, 'aff_face_pca'), feature_file, pca_file,
'-n',
str(self.ndims)
]
run_cmd(cmd)
return pca_file
def pca_projection(bin_dir, in_file_extract, in_file_pca, out_file):
verify_files_readable([in_file_extract, in_file_pca])
verify_files_writable(out_file)
run_cmd([join(bin_dir, 'pca_projection'), in_file_extract, in_file_pca,
out_file ])
def _run_extraction(self, infofile, extract_filename):
cmd = FACE_EXTRACT_CMD % (config.bin_dir(), infofile.name,
extract_filename)
run_cmd(cmd)
