def startpointCompare(queryVidName, dbVidName, dbStart):
"""
Startpoint compare take a frame number worth pursuing, and calculates the
average rmse value for the duration of the video starting at that point
"""
#Create the FFMPEG class variables
dbVid = FFMPEG_VideoReader(dbVidName)
queryVid = FFMPEG_VideoReader(queryVidName)
length = queryVid.nframes
# Skip to the startpoint frame
dbVid.skip_frames(dbStart)
frameD = dbVid.read_frame()
frameQ = queryVid.read_frame()
runAvg = 0
# Calculate the RMSE for each frame
for i in xrange(0, length):
runAvg += frame_rmse(frameD, frameQ)
frameQ = queryVid.read_frame()
frameD = dbVid.read_frame()
# Return the average RMSE score
return runAvg / length
def test_sequential_frame_pos():
"""test_video.mp4 contains 5 frames at 1 fps.
Each frame is 1x1 pixels and the sequence is Red, Green, Blue, Black, White.
The rgb values are not pure due to compression.
"""
reader = FFMPEG_VideoReader("media/test_video.mp4")
assert reader.pos == 1
# Get first frame
frame_1 = reader.get_frame(0)
assert reader.pos == 1
assert np.array_equal(frame_1, [[[254, 0, 0]]])
# Get a specific sequential frame
frame_2 = reader.get_frame(1)
assert reader.pos == 2
assert np.array_equal(frame_2, [[[0, 255, 1]]])
# Get next frame. Note `.read_frame()` instead of `.get_frame()`
frame_3 = reader.read_frame()
assert reader.pos == 3
assert np.array_equal(frame_3, [[[0, 0, 255]]])
# Skip a frame
skip_frame = reader.get_frame(4)
assert reader.pos == 5
assert np.array_equal(skip_frame, [[[255, 255, 255]]])
def get_video(vid_path, color=True, size=True):
"""Get video by given video path.
Parameters
----------
image_path : string
target image absolute path
color : bool
if color is True then return color frames with BGR encoding.
if color is False then return grey scale frames.
size : bool
if size is True then return the size of the frame.
if size is False then just return the frame.
Returns
-------
frames : list
a list of frames that contains the video
size : tuple
size of the frame (optional).
"""
vid_container = FFMPEG_VideoReader(vid_path)
frames = []
for i in range(vid_container.nframes):
frame_t = vid_container.read_frame()
frame_t = cv2.cvtColor(frame_t, cv2.COLOR_RGB2BGR)
if color is False:
frame_t = cv2.cvtColor(frame_t, cv2.COLOR_BGR2GRAY)
frames.append(frame_t)
if size is True:
return frames, frames[0].shape
else:
return frames
def test_moviepy(infile, out_dir):
import numpy as np
import pipi
from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader
from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter
chunksize = 128
vr = FFMPEG_VideoReader(infile)
w,h = vr.size
chunk = np.zeros((h, w, vr.depth, chunksize), dtype=np.uint8)
with pipi.Timer("mp read..."):
for i in range(chunksize):
frame = vr.read_frame()
chunk[...,i] = frame
vw = FFMPEG_VideoWriter(os.path.join(out_dir, "mp_ov.mkv"), (w,h), 30, codec="libx264", preset="fast", ffmpeg_params=["-crf", "0"])
with pipi.Timer("mp write..."):
for i in range(chunksize):
vw.write_frame(chunk[...,i])
vr.close()
vw.close()
def get_video(vid_path, color=True, size=True):
"""Get video by given video path.
Parameters
----------
image_path : string
target image absolute path
color : bool
if color is True then return color frames with BGR encoding.
if color is False then return grey scale frames.
size : bool
if size is True then return the size of the frame.
if size is False then just return the frame.
Returns
-------
frames : list
a list of frames that contains the video
size : tuple
size of the frame (optional).
"""
vid_container = FFMPEG_VideoReader(vid_path)
frames = []
for i in range(vid_container.nframes):
frame_t = vid_container.read_frame()
frame_t = cv2.cvtColor(frame_t, cv2.COLOR_RGB2BGR)
if color is False:
frame_t = cv2.cvtColor(frame_t, cv2.COLOR_BGR2GRAY)
frames.append(frame_t)
if size is True:
return frames, frames[0].shape
else:
return frames
def test_moviepy(infile, out_dir):
import numpy as np
import pipi
from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader
from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter
chunksize = 128
vr = FFMPEG_VideoReader(infile)
w,h = vr.size
chunk = np.zeros((h, w, vr.depth, chunksize), dtype=np.uint8)
with pipi.Timer("mp read..."):
for i in range(chunksize):
frame = vr.read_frame()
chunk[...,i] = frame
vw = FFMPEG_VideoWriter(os.path.join(out_dir, "mp_ov.mkv"), (w,h), 30, codec="libx264", preset="fast", ffmpeg_params=["-crf", "0"])
with pipi.Timer("mp write..."):
for i in range(chunksize):
vw.write_frame(chunk[...,i])
vr.close()
vw.close()
def comparechunk(queryVidName, dbVidName, dbStart, dbEnd, thresh):
"""
This function takes the names of the files to compare, and where in the
comparison file to begin checking from. Threshold is a user determined value
that they chose qualitatively, and the GUI turned into a quantitative number
Args: path to query video, path to comparison video, start frame number,
endframe number, callback to process results, RMSE threshold (a number from
0-255, realistically should be between 10-50)
"""
# Create the FFMPEG class variables
dbVid = FFMPEG_VideoReader(dbVidName)
queryVid = FFMPEG_VideoReader(queryVidName)
# Skip to the correct frames in the video
frameQ = queryVid.get_frame(0)
dbVid.skip_frames(dbStart)
frameD = dbVid.read_frame()
scores = []
# Compare the first frame in the query video to every frame in the chunk
belowThresh = []
for i in range(dbStart, dbEnd):
score = frame_rmse(frameQ, frameD)
# Immediately look at startframes below the threshold
if frame_rmse(frameQ, frameD) < thresh:
print "Found a frame below the threshold. Scanning sequence..."
score = startpointCompare(queryVidName, dbVidName, i)
if score < thresh and score is not None:
scores.append({
"Video Name": dbVidName,
"Timestamp": secondsToTimestamp(i / dbVid.fps),
"Frame Number": i,
"Score": score
})
return scores
else:
print "A sequence had a poor score of", score, ". Moving on..."
frameD = dbVid.read_frame()
return scores
def get_frames(video_path: str) -> List[np.ndarray]:
"""
Load frames from video.
:param video_path: path to video.
:return: loaded frames.
"""
video_reader = FFMPEG_VideoReader(video_path)
frames = []
for _ in tqdm(range(video_reader.nframes), desc='Getting video frames'):
frames.append(video_reader.read_frame())
return frames
def save_frames_from_vid(video_name, category_id, train_or_test, row=0):
make_dir_structure(row)
# Initialize FFMPEG_VideoReader
fvr = FFMPEG_VideoReader(filename=video_name)
fvr.initialize()
vid = os.path.split(video_name)[1]
for i in range(0, fvr.nframes):
frame_name = vid + '_' + str(i)
frame = fvr.read_frame()
imsave(os.path.join('trafficdb', 'eval_'+ str(row), train_or_test, str(category_id), frame_name + '.jpg'),
frame)
return True
def get_frames_from_vid(video_name, category_id):
# Initialize FFMPEG_VideoReader
fvr = FFMPEG_VideoReader(filename=video_name)
fvr.initialize()
shape_for_stack = (1, fvr.size[0], fvr.size[1], fvr.depth)
img_stack = np.zeros(shape_for_stack)
for i in range(0, fvr.nframes):
frame = fvr.read_frame()
frame = frame.reshape(shape_for_stack)
img_stack = np.vstack((img_stack, frame))
img_stack = img_stack[1:]
cat_stack = np.ones((len(img_stack), 1)) * category_id
return img_stack, cat_stack
if arguments_strVideoAudio:
writer = FFMPEG_VideoWriter(arguments_strVideoOut,
reader.size,
reader.fps * 2,
audiofile=arguments_strVideoAudio)
else:
writer = FFMPEG_VideoWriter(arguments_strVideoOut, reader.size,
reader.fps * 2)
if exists:
# Write frames that were already completed
print("Re-writing processed frames...")
printProgressBar(0, readerCont.nframes)
for x in range(0, readerCont.nframes):
writer.write_frame(readerCont.read_frame())
printProgressBar(x + 1, readerCont.nframes)
reader.skip_frames(startFrame)
readerCont.close()
print("Deleting temporary file(s)...")
shutil.rmtree(tempDir)
print("Processing resumed!")
print("")
totalFrames = reader.nframes
nextFrame = reader.read_frame()
startedTime = datetime.now()
class MovieEditor(object):
def __init__(self,
input,
output,
width=None,
height=None,
log_level=FFMPEG_LOGLEVEL):
self._reader = FFMPEG_VideoReader(input)
self._fps = self._reader.fps
# self.cur_frame = 1
# self._frame = None
self._querier = FfmpegQuerier()
self._info = self._querier(input)
self._duration = self._querier.duration
self.draw_dict = {}
self._resize = (int(width),
int(height)) if (width and height) else None
self._loglevel = log_level
self._output = output
self._tmp_file = self._make_tmp_file(input) if self._resize else None
self._writer = FFMPEG_VideoWriter(
self._tmp_file if self.need_resize else output,
size=self._reader.size,
fps=self._reader.fps)
@property
def need_resize(self):
return self._resize and list(self._resize) != list(self._reader.size)
# for resizing
def _make_tmp_file(self, input):
return '%s-%s.mp4' % (input, get_time())
def _remove_tmp_file(self):
if os.path.exists(self._tmp_file):
os.remove(self._tmp_file)
# approximate frame count
def get_total_frame(self):
return timestamp2frame(self._duration, self._fps)
# def seek_frame(self, index):
# self._reader.skip_frames(index - self.cur_frame)
# self._frame = self._reader.read_frame()
#
# def seek_timestamp(self, timestamp):
# index = FrameEditor.which_frame(timestamp, self._fps)
# self.seek_frame(index)
#
# def draw_anno(self, anno):
# if self._frame is not None:
# self._frame_editor.frame = self._frame
# self._frame_editor.draw_anno(anno)
def draw_anno_file(self, anno_filename):
# parse anno file
ax = AnnoXml()
ax.read(anno_filename)
ax.parse()
commands = ax.command
fps = self._fps
frame = self._reader.read_frame()
self._writer.write_frame(frame)
# total frame
total_frame = self.get_total_frame()
with click.progressbar(length=total_frame,
label='Processing...') as bar:
# current
frame_index = 1
bar.update(1)
for command in commands:
pageid = command.pageid
command_frame = CommandParser.get_frame_index(command, fps)
if pageid in self.draw_dict:
frame_editor = self.draw_dict[pageid]
else:
frame_editor = FrameEditor(fps=fps, pageid=pageid)
self.draw_dict[pageid] = frame_editor
# before
while frame_index < command_frame:
if not frame_editor.commands:
frame = self.read_frame()
self.write_frame(frame)
else:
frame = self.read_frame()
frame_editor.draw(frame)
self.write_frame(frame_editor.image)
frame_index += 1
bar.update(1)
# append
frame_editor.append_command(command)
# current
frame = self.read_frame()
frame_editor.draw(frame)
self.write_frame(frame_editor.image)
frame_index += 1
bar.update(1)
# write left frames
while frame_index < total_frame:
self.write_frame(self.read_frame())
frame_index += 1
bar.update(1)
# close stream
self.close()
# resize if needed
if self.need_resize:
self.resize()
self._remove_tmp_file()
def read_frame(self):
return self._reader.read_frame()
def write_frame(self, frame):
# if self._resize:
# if not isinstance(frame, Image.Image):
# frame = Image.fromarray(frame.astype('uint8'))
# Here's a bug.
# frame = frame.resize(self._resize, Image.ANTIALIAS)
self._writer.write_frame(frame)
def resize(self):
parameters = [
FFMPEG_FILE, '-i', self._tmp_file, '-s',
'{w}*{h}'.format(w=self._resize[0], h=self._resize[1]),
'-loglevel', self._loglevel, '-y', self._output
]
subprocess.call(parameters)
def close(self):
self._reader.close()
self._writer.close()
print "n_frames:", movie_reader.nframes
# the 1st frame of the saved video can't be directly read by movie_reader.read_frame(), I don't know why
# maybe it's a bug of anim.save, actually, if we look at the movie we get from anim.save
# we can easilly see that the 1st frame just close very soon.
# so I manually get it at time 0, this is just a trick, I think.
tmp_frame = movie_reader.get_frame(0)
[
movie_writer.write_frame(tmp_frame)
for _ in range(int(new_fps * play_slow_rate))
]
# for the above reason, we should read (movie_reader.nframes-1) frames so that the last frame is not
# read twice (not that get_frame(0) alread read once)
# However, I soon figure out that it should be (movie_reader.nframes-2). The details: we have actually
# 6 frames, but (print movie_reader.nframes) is 7. I read the first frame through movie_reader.get_frame(0)
# then are are 5 left. So I should use movie_reader.nframes - 2. Note that in fig1_pcm_fs2.py
# in the case of: original fps=1
# new_fps = 24, play_slow_rate = 1.5 the result is: 1st frame last 1.8s, others 1.5s, i.e., the 1st frame
# has more duration. This is messy.
for i in range(movie_reader.nframes - 2):
tmp_frame = movie_reader.read_frame()
[
movie_writer.write_frame(tmp_frame)
for _ in range(int(new_fps * play_slow_rate))
]
pass
movie_reader.close()
movie_writer.close()
os.remove(tmp_video_name)
# plt.show()
pass
def extract_frames(video_data,video_root, destination, num_frames, seed=0):
"""Extracts [num_frames] images from a video and stores """
np.random.seed(seed)
file = h5py.File(destination,'w')
X = None
for n, (video, metadata) in enumerate(video_data.items()):
crop = list(map(int, metadata['crop'].split(',')))
if crop is not None:
x1, y1, x2, y2 = crop
else:
x1,y1 = [0,0]
x2,y2 = [-1,-1]
video_path = video_root/Path(video)
print('Extracting {} frames with seed {} from {}'.format(num_frames, seed, video_path.name))
print(crop)
video_path = Path(video_path).resolve()
if not video_path.exists():
raise FileNotFoundError('Video "{}" not found.'.format(video_path))
# img_path = Path(destination).absolute()
# TODO: Exception handling for faulty videos
# Warning: Handles to VideoClip object are fickle, and getting them stuck is pretty easy.
# Best always handle them with a context manager.
clip = FFMPEG_VideoReader(str(video_path))# as clip:
print('Video duration: {} s, {} fps, uncropped frame size: {}'.format(clip.duration, clip.fps, clip.size))
# Crop frames
# print('Cropping box {}->{}'.format((x1, y1), (x2, y2)))
# clip = clip.crop(x1, y1, x2, y2)
num_frames_clip = int(clip.duration * clip.fps)
#padding = int(math.ceil(math.log10(num_frames_clip))) # file name padding
# print('Storing images in {}'.format(img_path))
# Grab frames from video and store as png file
frame_indices = sorted(np.random.randint(0, max(1,num_frames_clip-11), num_frames))
for idx in tqdm(np.diff([0]+frame_indices)):
print(idx)
image = []
clip.skip_frames(idx)
for i in range(10):
frame = (clip.read_frame())#idx+i / clip.fps))
# print(image)
image.append(frame[y1:y2,x1:x2,:])
if X is None:
X = file.create_dataset('X',(1,10,y2-y1,x2-x1,3),
chunks=(1,10,y2-y1,x2-x1,3),
maxshape=(None,10,y2-y1,x2-x1,3),
compression="gzip", dtype = 'i')
else:
X.resize((X.shape[0]+1,)+X.shape[1:])
X[-1,:,:,:,:]=image
# print(X[-1,:,:,:,:])
file.close()
def __getitem__(self, index):
# Find the video first.
vid = np.histogram(index, self.sums)
assert (np.sum(vid[0]) == 1)
vid = np.where(vid[0] > 0)[0][0]
v_fmax = len(self.gt_list[vid])
vframe = index - self.sums[vid]
#vframes = [min(vframe + i, len(self.gt_list[vid])- 1) for i in np.arange(0, 1 +10*self.multi_frame, 10)]
#vframes = [min(vframe, len(self.gt_list[vid])- 1)]
#cap = self.video_list[vid]
imgs = []
if self.prod_Img:
cap = FFMPEG_VideoReader(self.video_list[vid])
cap.initialize()
for i in range(self.multi_frame):
if i == 0:
img = cap.get_frame(vframe / cap.fps)
else:
cap.skip_frames(n=9)
img = cap.read_frame()
img = Image.fromarray(img)
if self.transform is not None:
img = self.transform(img)
imgs.append(img)
'''
for v in vframes:
#cap = cv2.VideoCapture(self.video_list[vid])
#assert cap.isOpened() == True, 'The Video cannot be read:{}'.format(self.video_list[vid])
#cap.set(1, v)
#ret, frame= cap.read()
img = cap.get_frame(v)
#assert ret == True, 'Cannot Load this frame{}:{}'.format(self.video_list[vid], v)
#cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img = Image.fromarray(img)
if self.transform is not None:
img = self.transform(img)
imgs.append(img)
'''
imgs = [img.unsqueeze(0) for img in imgs]
imgs = torch.cat(imgs, 0)
text = []
if self.prod_Text:
text = self.text_list[vid][
min(vframe + self.text_delay, len(self.text_list[vid])
):min(vframe + self.text_window +
self.text_delay, len(self.text_list[vid]))]
text = '\n'.join(text)
gt = self.gt_list[vid][vframe]
if (len(text) == 0):
text = ' '
#text = text_util.lineToTensor(text)
return imgs, text, gt
frame = gui.resize(frame, (400, 300), ratio_keep=True)
print frame.shape
if option == "test-dict-compare":
para_dict_old = {}
para_dict_old["a"] = 1
para_dict_old["b"] = 2
para_dict_new = {}
para_dict_new["a"] = 1
para_dict_new["b"] = 2
print retina.compare_para_dict(para_dict_old, para_dict_new)
if option == "test-setup-function":
eye = retina.init_retina((300, 200))
print type(eye.setupOPLandIPLParvoChannel)
print type(eye.setupIPLMagnoChannel)
print eye.getInputSize()
if option == "test-movie-py":
video = FFMPEG_VideoReader("./simretina/retina-data/HorseRiding.avi")
frame = video.read_frame()
for i in range(video.nframes):
frame = video.read_frame()
cv2.imshow("test", frame)
cv2.waitKey(0)
if True: tensorInputFirst.cpu() tensorInputSecond.cpu() tensorOutput.cpu() # end #end tensorOutput = torch.FloatTensor() if arguments_strVideo and arguments_strVideoOut: # Process video reader = FFMPEG_VideoReader(arguments_strVideo, False) writer = FFMPEG_VideoWriter(arguments_strVideoOut, reader.size, reader.fps*2) reader.initialize() nextFrame = reader.read_frame() for x in range(0, reader.nframes): firstFrame = nextFrame nextFrame = reader.read_frame() tensorInputFirst = torch.FloatTensor(numpy.rollaxis(firstFrame[:,:,::-1], 2, 0) / 255.0) tensorInputSecond = torch.FloatTensor(numpy.rollaxis(nextFrame[:,:,::-1], 2, 0) / 255.0) process(tensorInputFirst, tensorInputSecond, tensorOutput) writer.write_frame(firstFrame) writer.write_frame((numpy.rollaxis(tensorOutput.clamp(0.0, 1.0).numpy(), 0, 3)[:,:,::-1] * 255.0).astype(numpy.uint8)) #end writer.write_frame(nextFrame) writer.close() else: # Process image tensorInputFirst = torch.FloatTensor(numpy.rollaxis(numpy.asarray(PIL.Image.open(arguments_strFirst))[:,:,::-1], 2, 0).astype(numpy.float32) / 255.0) tensorInputSecond = torch.FloatTensor(numpy.rollaxis(numpy.asarray(PIL.Image.open(arguments_strSecond))[:,:,::-1], 2, 0).astype(numpy.float32) / 255.0)
class FinalMerger(object):
"""
merge 3 mp4 files to one mp4 file,
in other words, 3to1.
layout:
---------------
| | |
| | grf |
| swf ------
| | |
| | chat |
---------------
"""
def __init__(self, swf_mp4, grf_mp4, chat_mp4, output, fps=10):
self._swf_mp4 = swf_mp4
self._grf_mp4 = grf_mp4
self._chat_mp4 = chat_mp4
self._fq = FfmpegQuerier()
self._swf_reader = FFMPEG_VideoReader(swf_mp4)
self._grf_reader = FFMPEG_VideoReader(grf_mp4)
self._chat_reader = FFMPEG_VideoReader(chat_mp4)
self._output = output
self._fps = fps
self._output_size = self.cal_output_size()
self._writer = FFMPEG_VideoWriter(output,
self._output_size,
fps,
audiofile=grf_mp4)
def get_duration(self, filename) -> float:
self._fq(filename)
return float(self._fq.duration)
def get_size(self, filename) -> (int, int):
self._fq(filename)
return self._fq.size
def cal_output_size(self):
swf_size = self.get_size(self._swf_mp4)
grf_size = self.get_size(self._grf_mp4)
return swf_size[0] + grf_size[0], swf_size[1]
def _merge_frame(self, swf_frame, grf_frame, chat_frame):
sf = swf_frame.astype('uint8')
gf = grf_frame.astype('uint8')
cf = chat_frame.astype('uint8')
return np.column_stack([sf, np.row_stack([gf, cf])])
def merge(self):
# get durations
durations = list(
map(lambda x: self.get_duration(x),
[self._swf_mp4, self._grf_mp4, self._chat_mp4]))
max_duration = max(durations)
# max frames
max_frames = int(max_duration * self._fps)
with click.progressbar(length=max_frames,
label='Processing...') as bar:
index = 0
while index < max_frames:
sf = self.read_swf_frame()
gf = self.read_grf_frame()
cf = self.read_chat_frame()
frame = self._merge_frame(sf, gf, cf)
self.write_frame(frame)
index += 1
bar.update(1)
def read_swf_frame(self):
return self._swf_reader.read_frame()
def read_grf_frame(self):
return self._grf_reader.read_frame()
def read_chat_frame(self):
return self._chat_reader.read_frame()
def write_frame(self, frame):
self._writer.write_frame(frame)
def close(self):
for one in (self._swf_reader, self._grf_reader, self._chat_reader,
self._writer):
one.close()