def main(path, output_dir):
try:
os.makedirs(output_dir)
except OSError:
pass
kf = keyframe.Meta(
[keyframe.SURF(min_diff=10),
keyframe.Histogram(min_diff=10)],
min_diff=2)
prev_frame = None
for (frame_num, frame_time,
frame), iskeyframe in kf(viderator.frame_iter(path)):
if prev_frame is not None:
if iskeyframe:
cv2.imwrite(output_dir + '/%.8d-.jpg' % (frame_num - 1),
prev_frame)
cv2.imwrite(output_dir + '/%.8d+.jpg' % frame_num, frame)
prev_frame = frame
import matplotlib
matplotlib.use('agg')
import pylab
pylab.figure(1)
pylab.clf()
pylab.plot(kf.get_scores())
pylab.savefig('matches_fig.png')
def mapper(videohash, metadata):
print('videohash[%s]' % videohash)
print('hdfs_path[%s]' % metadata['hdfs_path'])
print 'mapper', videohash
filename = 'hardcodedvideo.' + metadata['extension']
#print filename, metadata.keys()
try:
picarus.io._record_to_file(metadata, filename)
except IOError:
hadoopy.counter('INPUT_ERROR', 'REMOTE READ FAILED')
return
min_interval = float(os.environ['MIN_INTERVAL'])
resolution = float(os.environ['RESOLUTION'])
try:
iter1 = lambda: viderator.frame_iter(
filename, frame_skip=5, frozen=True)
iter2 = lambda: viderator.convert_video_ffmpeg(
filename, frame_skip=5, frozen=True)
kf = keyframe.Histogram(min_interval)
# Do this instead of 'return' in order to keep the tempfile around
try:
for k, v in keyframes(iter1, iter2, metadata, kf, resolution):
#print 'yield', k
yield k, v
except:
hadoopy.counter('INPUT_ERROR', 'VIDEO_READ_ERROR')
return
finally:
os.remove(filename)
def mapper(videohash, metadata):
print('videohash[%s]' % videohash)
print('hdfs_path[%s]' % metadata['hdfs_path'])
print 'mapper', videohash
filename = 'hardcodedvideo.' + metadata['extension']
#print filename, metadata.keys()
try:
picarus.io._record_to_file(metadata, filename)
except IOError:
hadoopy.counter('INPUT_ERROR', 'REMOTE READ FAILED')
return
min_interval = float(os.environ['MIN_INTERVAL'])
resolution = float(os.environ['RESOLUTION'])
try:
iter1 = lambda : viderator.frame_iter(filename, frame_skip=5, frozen=True)
iter2 = lambda : viderator.convert_video_ffmpeg(filename, frame_skip=5, frozen=True)
kf = keyframe.Histogram(min_interval)
# Do this instead of 'return' in order to keep the tempfile around
try:
for k, v in keyframes(iter1, iter2, metadata, kf, resolution):
#print 'yield', k
yield k, v
except:
hadoopy.counter('INPUT_ERROR', 'VIDEO_READ_ERROR')
return
finally:
os.remove(filename)
def register_frames(row):
import tempfile
import viderator
import cv2
import distpy
import time
hamming = distpy.Hamming()
thrift = hadoopy_hbase.connect()
fp = tempfile.NamedTemporaryFile()
def get_column(column):
return thrift.getRowWithColumns('videos', row,
[column])[0].columns[column].value
video_chunks = int(get_column('meta:video_chunks'))
for x in range(video_chunks):
fp.write(get_column('data:video-%d' % x))
fp.flush()
brisk = cv2.BRISK(40, 4, 1.) # TODO: Get from model
mask = None
prev_descs = None
prev_points = None
match_thresh = 75
min_inliers = 10
frames_matched = []
for frame_num, frame_time, frame in viderator.frame_iter(fp.name,
frame_skip=3):
matched = False
st = time.time()
if mask is None:
mask = np.ones((frame.shape[0], frame.shape[1]))
points, descs = brisk.detectAndCompute(frame, mask)
points = np.array([x.pt for x in points])
#print((frame_num, points, descs))
if prev_descs is not None:
matches = (hamming.cdist(prev_descs, descs) <
match_thresh).nonzero()
print(matches)
a = np.array(prev_points[matches[0], :], dtype=np.float32)
b = np.array(points[matches[1], :], dtype=np.float32)
#print(a)
#print(b)
if a.shape[0] >= min_inliers and b.shape[0] >= min_inliers:
h, i = cv2.findHomography(a, b, cv2.RANSAC)
if np.sum(i) >= min_inliers:
matched = True
print((h, i))
frames_matched.append(matched)
prev_descs = descs
prev_points = points
print(time.time() - st)
print(matched)
def test_basic(self):
import viderator
import time
import numpy as np
st = time.time()
for frame_num, frame_time, frame in viderator.frame_iter('HVC236624.mp4'):
self.assertTrue(isinstance(frame_num, int))
self.assertTrue(isinstance(frame_time, float))
self.assertTrue(isinstance(frame, np.ndarray))
if frame_num > 1000:
break
print(frame_num)
print((time.time() - st) / float(frame_num))
def register_frames(row):
import tempfile
import viderator
import cv2
import distpy
import time
hamming = distpy.Hamming()
thrift = hadoopy_hbase.connect()
fp = tempfile.NamedTemporaryFile()
def get_column(column):
return thrift.getRowWithColumns('videos', row,
[column])[0].columns[column].value
video_chunks = int(get_column('meta:video_chunks'))
for x in range(video_chunks):
fp.write(get_column('data:video-%d' % x))
fp.flush()
brisk = cv2.BRISK(40, 4, 1.) # TODO: Get from model
mask = None
prev_descs = None
prev_points = None
match_thresh = 75
min_inliers = 10
frames_matched = []
for frame_num, frame_time, frame in viderator.frame_iter(fp.name, frame_skip=3):
matched = False
st = time.time()
if mask is None:
mask = np.ones((frame.shape[0], frame.shape[1]))
points, descs = brisk.detectAndCompute(frame, mask)
points = np.array([x.pt for x in points])
#print((frame_num, points, descs))
if prev_descs is not None:
matches = (hamming.cdist(prev_descs, descs) < match_thresh).nonzero()
print(matches)
a = np.array(prev_points[matches[0], :], dtype=np.float32)
b = np.array(points[matches[1], :], dtype=np.float32)
#print(a)
#print(b)
if a.shape[0] >= min_inliers and b.shape[0] >= min_inliers:
h, i = cv2.findHomography(a, b, cv2.RANSAC)
if np.sum(i) >= min_inliers:
matched = True
print((h, i))
frames_matched.append(matched)
prev_descs = descs
prev_points = points
print(time.time() - st)
print(matched)
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename), value) where value is a dict
with contents
prev_frame_time:
prev_frame_num:
prev_frame:
frame_time:
frame_num:
frame:
"""
ext = '.' + event_filename[1].rsplit('.')[1]
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
prev_frame_time = None
prev_frame_num = None
prev_frame = None
try:
for (frame_num, frame_time, frame), iskeyframe in self.kf(
viderator.frame_iter(fp.name,
frame_skip=self.frame_skip,
frozen=True)):
if self.max_time < frame_time:
break
if iskeyframe and prev_frame is not None:
yield event_filename, {
'prev_frame_time': prev_frame_time,
'prev_frame_num': prev_frame_num,
'prev_frame':
imfeat.image_tostring(prev_frame, 'JPEG'),
'frame_time': frame_time,
'frame_num': frame_num,
'frame': imfeat.image_tostring(frame, 'JPEG')
}
prev_frame_num = frame_num
prev_frame = frame
except Exception, e:
print(e)
hadoopy.counter('VIDEO_ERROR', 'FFMPEGCantParse')
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename, frame_num, frame_time), frame_data)
"""
ext = '.' + event_filename[1].rsplit('.')[1]
event, filename = event_filename
heap = [(float('-inf'), None)] * self.max_outputs
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
sys.stderr.write('Prevideo\n')
try:
for frame_num, frame_time, frame in viderator.frame_iter(fp.name,
frozen=True,
frame_skip=self.frame_skip):
sys.stderr.write('FrameNum[%d]\n' % frame_num)
if frame_num >= self.max_frames_per_video:
break
frame_orig = frame
if self.remove_bars:
sz = self.remove_bars.find_bars(frame)
frame = frame[sz[0]:sz[1], sz[2]:sz[3], :]
if not frame.size: # Empty
continue
st = time.time()
c = self._feat(frame)[0]
sys.stderr.write('FrameTime[%f]\n' % (time.time() - st))
print('FrameTime[%f]' % (time.time() - st))
if c > heap[0][0]:
if self.output_frame:
heapq.heappushpop(heap,
(c, ((event, filename, frame_num, frame_time), imfeat.image_tostring(frame_orig, 'JPEG'))))
else:
heapq.heappushpop(heap,
(c, ((event, filename, frame_num, frame_time), '')))
except IOError:
hadoopy.counter('PICARUS', 'CantProcessVideo')
for x in heap[-self.max_outputs_per_video:]:
heapq.heappushpop(self.heap, x)
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename), value) where value is a dict
with contents
prev_frame_time:
prev_frame_num:
prev_frame:
frame_time:
frame_num:
frame:
"""
ext = '.' + event_filename[1].rsplit('.')[1]
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
prev_frame_time = None
prev_frame_num = None
prev_frame = None
try:
for (frame_num, frame_time, frame), iskeyframe in self.kf(viderator.frame_iter(fp.name,
frame_skip=self.frame_skip,
frozen=True)):
if self.max_time < frame_time:
break
if iskeyframe and prev_frame is not None:
yield event_filename, {'prev_frame_time': prev_frame_time,
'prev_frame_num': prev_frame_num,
'prev_frame': imfeat.image_tostring(prev_frame, 'JPEG'),
'frame_time': frame_time,
'frame_num': frame_num,
'frame': imfeat.image_tostring(frame, 'JPEG')}
prev_frame_num = frame_num
prev_frame = frame
except Exception, e:
print(e)
hadoopy.counter('VIDEO_ERROR', 'FFMPEGCantParse')
def main(path, output_dir):
try:
os.makedirs(output_dir)
except OSError:
pass
kf = keyframe.Meta([keyframe.SURF(min_diff=10), keyframe.Histogram(min_diff=10)], min_diff=2)
prev_frame = None
for (frame_num, frame_time, frame), iskeyframe in kf(viderator.frame_iter(path)):
if prev_frame is not None:
if iskeyframe:
cv2.imwrite(output_dir + '/%.8d-.jpg' % (frame_num - 1), prev_frame)
cv2.imwrite(output_dir + '/%.8d+.jpg' % frame_num, frame)
prev_frame = frame
import matplotlib
matplotlib.use('agg')
import pylab
pylab.figure(1)
pylab.clf()
pylab.plot(kf.get_scores())
pylab.savefig('matches_fig.png')
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename, frame_num, frame_time), frame_data)
"""
ext = '.' + event_filename[1].rsplit('.')[1]
event, filename = event_filename
out_count = 0
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
try:
for frame_num, frame_time, frame in viderator.frame_iter(
fp.name, frozen=True, frame_skip=self.frame_skip):
if frame_num >= self.max_frames_per_video:
break
frame_orig = frame
if self.remove_bars:
sz = self.remove_bars.find_bars(frame)
frame = frame[sz[0]:sz[1], sz[2]:sz[3], :]
if not frame.size: # Empty
continue
if self._feat.predict(frame):
if self.output_frame:
yield (event, filename, frame_num,
frame_time), imfeat.image_tostring(
frame_orig, 'JPEG')
else:
yield (event, filename, frame_num, frame_time), ''
out_count += 1
if out_count >= self.max_outputs_per_video:
break
except IOError:
hadoopy.counter('PICARUS', 'CantProcessVideo')
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename, frame_num, frame_time), frame_data)
"""
ext = '.' + event_filename[1].rsplit('.')[1]
event, filename = event_filename
out_count = 0
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
try:
for frame_num, frame_time, frame in viderator.frame_iter(fp.name,
frozen=True,
frame_skip=self.frame_skip):
if frame_num >= self.max_frames_per_video:
break
frame_orig = frame
if self.remove_bars:
sz = self.remove_bars.find_bars(frame)
frame = frame[sz[0]:sz[1], sz[2]:sz[3], :]
if not frame.size: # Empty
continue
if self._feat.predict(frame):
if self.output_frame:
yield (event, filename, frame_num, frame_time), imfeat.image_tostring(frame_orig, 'JPEG')
else:
yield (event, filename, frame_num, frame_time), ''
out_count += 1
if out_count >= self.max_outputs_per_video:
break
except IOError:
hadoopy.counter('PICARUS', 'CantProcessVideo')
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of (event, (filename, array)) where
array is F x (P + 2) where P is the number of predicates, F
is the number of frames. The last two columns are frame_num
and frame_time.
"""
print(os.environ['map_input_file'])
print(event_filename)
ext = '.' + event_filename[1].rsplit('.')[1]
event, filename = event_filename
out = []
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
fp.write(video_data)
fp.flush()
try:
for frame_num, frame_time, frame in viderator.frame_iter(fp.name,
frozen=True,
frame_skip=self.frame_skip):
if frame_num >= self.max_frames_per_video:
break
if self.remove_bars:
sz = self.remove_bars.find_bars(frame)
frame = frame[sz[0]:sz[1], sz[2]:sz[3], :]
out.append([x.predict(frame) for x in self._feats] + [frame_num, frame_time])
except IOError:
hadoopy.counter('PICARUS', 'CantProcessVideo')
hadoopy.counter('SkippingTaskCounters', 'MapProcessedRecords')
return
yield event, (filename, np.asfarray(out))
hadoopy.counter('SkippingTaskCounters', 'MapProcessedRecords')
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename), features) where features is a dict
frame_features: List of frame features
file_size: Size in bytes
where each frame feature is a dictionary of
frame_time: Time in seconds
frame_num: Frame number
prev_frame_num: Previous frame number (useful if there is a frame skip)
keyframe: Boolean True/False
surf: List of surf points (see impoint)
face_widths:
face_heights:
predictions: Dictionary of predictions
"""
sys.stderr.write('In Raw:%s\n' % str(event_filename))
print(event_filename)
ext = '.' + event_filename[1].rsplit('.')[1]
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
with self.timer('Writing video data'):
fp.write(video_data)
fp.flush()
kf = keyframe.DecisionTree(min_interval=0)
kf.load()
prev_frame = None
prev_frame_num = 0
all_out = []
sz = len(video_data)
self.timer.start('KF')
try:
for (frame_num, frame_time, frame), iskeyframe in kf(viderator.frame_iter(fp.name,
frozen=True)):
hadoopy.counter('RawFeatures', 'NumFrames')
self.timer.stop('KF')
print(frame_time)
if frame_num > self._max_frames:
break
if frame_num % 100 == 0:
with self.timer('Computing face features'):
faces = _detect_faces(imfeat.convert_image(frame, [('opencv', 'gray', 8)]),
self.cascade)
else:
faces = {}
out = {'frame_time': frame_time, 'frame_num': frame_num,
'prev_frame_num': prev_frame_num, 'keyframe': iskeyframe,
'surf': kf.prev_vec['surf']}
if faces: # If any faces
face_heights = np.array([x[0][3] for x in faces]) / float(frame.height)
face_widths = np.array([x[0][2] for x in faces]) / float(frame.width)
out['face_widths'] = face_widths
out['face_heights'] = face_heights
# Output the cur and previous frames if this is a keyframe
if iskeyframe and np.random.random() < self._frame_output_prob:
out['prev_frame'] = cv_to_jpg(prev_frame)
out['frame'] = cv_to_jpg(frame)
# Compute scene features
with self.timer('Computing scene classifier features'):
frame_res = cv.fromarray(cv2.resize(np.asarray(cv.GetMat(frame)), (self._image_width, self._image_height)))
feature = self._feat(frame_res)
out['predictions'] = dict((classifier_name, classifier.predict(feature))
for classifier_name, classifier in self._classifiers)
# Output JPEG with match lines from the SURF feature
if np.random.random() < self._match_line_prob and prev_frame:
out['surf_image'] = cv_to_jpg(plot_matches(prev_frame, kf.surf_debug['matches'], kf.surf_debug['points0'],
kf.surf_debug['points1'], max_feat_width=kf.max_feat_width))
# Output data buffer
all_out.append(out)
if len(all_out) >= self._block_size:
with self.timer('Yield'):
yield event_filename, {'frame_features': all_out,
'file_size': sz}
all_out = []
prev_frame = frame
prev_frame_num = frame_num
self.timer.start('KF')
except viderator.FPSParseException: # NOTE(brandyn): This will disregard videos with this error
hadoopy.counter('SkippedVideos', 'FPSParseException')
return
if all_out:
with self.timer('Yield'):
yield event_filename, {'frame_features': all_out,
'file_size': sz}
def map(self, event_filename, video_data):
"""
Args:
event_filename: Tuple of (event, filename)
video_data: Binary video data
Yields:
A tuple in the form of ((event, filename), features) where features is a dict
frame_features: List of frame features
file_size: Size in bytes
where each frame feature is a dictionary of
frame_time: Time in seconds
frame_num: Frame number
prev_frame_num: Previous frame number (useful if there is a frame skip)
keyframe: Boolean True/False
surf: List of surf points (see impoint)
face_widths:
face_heights:
predictions: Dictionary of predictions
"""
sys.stderr.write('In Raw:%s\n' % str(event_filename))
print(event_filename)
ext = '.' + event_filename[1].rsplit('.')[1]
with tempfile.NamedTemporaryFile(suffix=ext) as fp:
with self.timer('Writing video data'):
fp.write(video_data)
fp.flush()
kf = keyframe.DecisionTree(min_interval=0)
kf.load()
prev_frame = None
prev_frame_num = 0
all_out = []
sz = len(video_data)
self.timer.start('KF')
try:
for (frame_num, frame_time, frame), iskeyframe in kf(
viderator.frame_iter(fp.name, frozen=True)):
hadoopy.counter('RawFeatures', 'NumFrames')
self.timer.stop('KF')
print(frame_time)
if frame_num > self._max_frames:
break
if frame_num % 100 == 0:
with self.timer('Computing face features'):
faces = _detect_faces(
imfeat.convert_image(frame,
[('opencv', 'gray', 8)]),
self.cascade)
else:
faces = {}
out = {
'frame_time': frame_time,
'frame_num': frame_num,
'prev_frame_num': prev_frame_num,
'keyframe': iskeyframe,
'surf': kf.prev_vec['surf']
}
if faces: # If any faces
face_heights = np.array([x[0][3] for x in faces
]) / float(frame.height)
face_widths = np.array([x[0][2] for x in faces
]) / float(frame.width)
out['face_widths'] = face_widths
out['face_heights'] = face_heights
# Output the cur and previous frames if this is a keyframe
if iskeyframe and np.random.random(
) < self._frame_output_prob:
out['prev_frame'] = cv_to_jpg(prev_frame)
out['frame'] = cv_to_jpg(frame)
# Compute scene features
with self.timer('Computing scene classifier features'):
frame_res = cv.fromarray(
cv2.resize(
np.asarray(cv.GetMat(frame)),
(self._image_width, self._image_height)))
feature = self._feat(frame_res)
out['predictions'] = dict(
(classifier_name, classifier.predict(feature)) for
classifier_name, classifier in self._classifiers)
# Output JPEG with match lines from the SURF feature
if np.random.random(
) < self._match_line_prob and prev_frame:
out['surf_image'] = cv_to_jpg(
plot_matches(prev_frame,
kf.surf_debug['matches'],
kf.surf_debug['points0'],
kf.surf_debug['points1'],
max_feat_width=kf.max_feat_width))
# Output data buffer
all_out.append(out)
if len(all_out) >= self._block_size:
with self.timer('Yield'):
yield event_filename, {
'frame_features': all_out,
'file_size': sz
}
all_out = []
prev_frame = frame
prev_frame_num = frame_num
self.timer.start('KF')
except viderator.FPSParseException: # NOTE(brandyn): This will disregard videos with this error
hadoopy.counter('SkippedVideos', 'FPSParseException')
return
if all_out:
with self.timer('Yield'):
yield event_filename, {
'frame_features': all_out,
'file_size': sz
}
def test_nonexist(self):
import viderator
self.assertRaises(IOError, viderator.frame_iter('sdklfjslkdfjsjdfkjskdfjsdjfkjskdfjskdfjksjfdj.IDONTEXIST.avi',).next)
