def adjustPositionsFFMPEG(meta, video_frames, hits):
rate = ffmpeg_api.get_video_frame_rate_from_meta([meta],
[video_frames])
aptime = 0
lasttime = 0
hitspos = 0
start_mask = None
for pos in range(0, len(video_frames)):
aptime = get_frame_time(video_frames[pos], aptime, rate)
while hitspos < len(hits) and aptime > hits[hitspos][0]:
mask = hits[hitspos][2]
element = hits[hitspos][1]
error = abs(aptime - hits[hitspos][0])
if element == 'starttime':
update_segment(mask,
starttime=lasttime,
startframe=pos,
error=error +
get_error_from_segment(mask))
start_mask = mask
else:
# for error, only record the error if not recorded
update_segment(
mask,
endtime=lasttime,
endframe=pos,
error=(error if start_mask != mask else 0) +
get_error_from_segment(mask))
hitspos += 1
lasttime = aptime
return mask
def adjustPositions(video_file, hits):
# used if variable frame rate
frmcnt = 0
hitspos = 0
last = 0
cap = cv2api_delegate.videoCapture(video_file)
try:
while cap.grab() and hitspos < len(hits):
frmcnt += 1
aptime = cap.get(cv2api_delegate.prop_pos_msec)
while hitspos < len(hits) and aptime > hits[hitspos][0]:
mask = hits[hitspos][2]
element = hits[hitspos][1]
error = max(abs(last - hits[hitspos][0]),
abs(aptime - hits[hitspos][0]))
if element == 'starttime':
update_segment(mask,
starttime=last,
startframe=frmcnt,
error=error)
else:
update_segment(mask,
endtime=last,
endframe=frmcnt,
error=max(
error,
get_error_from_segment(mask)))
hitspos += 1
last = aptime
finally:
cap.release()
return mask
def warpMask(self,
video_masks,
source,
target,
expectedType='video',
inverse=False,
useFFMPEG=False):
"""
Tranform masks when the frame rate has changed.
:param video_masks: ithe set of video masks to walk through and transform
:param expectedType:
:param video_masks:
:return: new set of video masks
"""
edge = self.graph.get_edge(source, target)
meta_i, frames_i = self.getVideoMeta(source,
show_streams=True,
media_types=[expectedType])
meta_o, frames_o = self.getVideoMeta(target,
show_streams=True,
media_types=[expectedType])
indices_i = ffmpeg_api.get_stream_indices_of_type(meta_i, expectedType)
indices_o = ffmpeg_api.get_stream_indices_of_type(meta_o, expectedType)
if not indices_i or not indices_o:
return video_masks
index_i = indices_i[0]
index_o = indices_o[0]
isVFR = ffmpeg_api.is_vfr(meta_i[index_i]) or ffmpeg_api.is_vfr(
meta_o[index_o])
result = self.getChangeInFrames(edge,
meta_i[index_i],
meta_o[index_o],
source,
target,
expectedType=expectedType)
if result is None:
return video_masks
sourceFrames, sourceTime, targetFrames, targetTime, sourceRate, targetRate = result
if sourceFrames == targetFrames and int(sourceTime * 100) == int(
targetTime * 100):
return video_masks
dropRate = sourceFrames / float(
sourceFrames -
targetFrames) if sourceFrames != targetFrames else sourceFrames + 1
#if sourceFrames > targetFrames else targetFrames / float(sourceFrames - targetFrames)
def apply_change(existing_value,
orig_rate,
final_rate,
inverse=False,
round_value=True,
min_value=0,
upper_bound=False):
# if round_value, return a tuple of value plus rounding error
import math
multiplier = -1.0 if inverse else 1.0
adjustment = existing_value * math.pow(final_rate / orig_rate,
multiplier)
if round_value:
v = max(
min(round(adjustment), final_rate)
if upper_bound else round(adjustment), min_value)
e = abs(adjustment - v)
return int(v), e
return max(
min(adjustment, final_rate) if upper_bound else adjustment,
min_value)
def adjustPositionsFFMPEG(meta, video_frames, hits):
rate = ffmpeg_api.get_video_frame_rate_from_meta([meta],
[video_frames])
aptime = 0
lasttime = 0
hitspos = 0
start_mask = None
for pos in range(0, len(video_frames)):
aptime = get_frame_time(video_frames[pos], aptime, rate)
while hitspos < len(hits) and aptime > hits[hitspos][0]:
mask = hits[hitspos][2]
element = hits[hitspos][1]
error = abs(aptime - hits[hitspos][0])
if element == 'starttime':
update_segment(mask,
starttime=lasttime,
startframe=pos,
error=error +
get_error_from_segment(mask))
start_mask = mask
else:
# for error, only record the error if not recorded
update_segment(
mask,
endtime=lasttime,
endframe=pos,
error=(error if start_mask != mask else 0) +
get_error_from_segment(mask))
hitspos += 1
lasttime = aptime
return mask
def adjustPositions(video_file, hits):
# used if variable frame rate
frmcnt = 0
hitspos = 0
last = 0
cap = cv2api_delegate.videoCapture(video_file)
try:
while cap.grab() and hitspos < len(hits):
frmcnt += 1
aptime = cap.get(cv2api_delegate.prop_pos_msec)
while hitspos < len(hits) and aptime > hits[hitspos][0]:
mask = hits[hitspos][2]
element = hits[hitspos][1]
error = max(abs(last - hits[hitspos][0]),
abs(aptime - hits[hitspos][0]))
if element == 'starttime':
update_segment(mask,
starttime=last,
startframe=frmcnt,
error=error)
else:
update_segment(mask,
endtime=last,
endframe=frmcnt,
error=max(
error,
get_error_from_segment(mask)))
hitspos += 1
last = aptime
finally:
cap.release()
return mask
new_mask_set = []
hits = []
# First adjust all the frame and time references by the total change in the video.
# In most cases, the length of the video in time changes by a small amount which is distributed
# across all the masks
for mask_set in video_masks:
if 'type' in mask_set and mask_set['type'] != expectedType:
new_mask_set.append(mask_set)
continue
#these are initial estimates
startframe, error_start = apply_change(
get_start_frame_from_segment(mask_set),
float(sourceFrames),
float(targetFrames),
inverse=inverse,
round_value=True,
min_value=1)
endframe, error_end = apply_change(
get_end_frame_from_segment(mask_set),
float(sourceFrames),
float(targetFrames),
inverse=inverse,
min_value=1,
round_value=True,
upper_bound=True)
endtime = apply_change(get_end_time_from_segment(mask_set),
float(sourceTime),
targetTime,
inverse=inverse,
round_value=False)
starttime = apply_change(get_start_time_from_segment(mask_set),
sourceTime,
targetTime,
inverse=inverse,
round_value=False,
upper_bound=True)
try:
if endframe == int(getValue(meta_o[index_o], 'nb_frames', 0)) and \
float(getValue(meta_o[index_o], 'duration', 0)) > 0:
endtime = float(getValue(meta_o[index_o], 'duration',
0)) * 1000.0 - (1000.0 /
targetRate)
elif endtime > targetTime and endframe > targetFrames:
message = '{} exceeded target time of {} for {}'.format(
sourceTime, target, targetTime)
if (endtime - targetTime) > 300:
logging.getLogger('maskgen').error(message)
else:
logging.getLogger('maskgen').warn(message)
endtime = targetTime - (1000.0 / targetRate)
endframe = targetFrames
except:
pass
change = create_segment(
rate=sourceRate if inverse else targetRate,
type=get_type_of_segment(mask_set),
starttime=starttime,
startframe=startframe,
error=get_error_from_segment(mask_set) +
(max(error_start, error_end) / targetRate * 1000.0),
endtime=endtime,
endframe=endframe,
videosegment=get_file_from_segment(mask_set))
if not isVFR:
# in this case, we trust the time invariance, updating frames
recalculate_frames_for_segment(change)
# then we reupdate time to match the frames
recalculate_times_for_segment(change)
new_mask_set.append(change)
hits.append(
(get_start_time_from_segment(change), 'starttime', change))
hits.append((get_end_time_from_segment(change), 'endime', change))
# only required when one of the two videos is variable rate
hits = sorted(hits)
if isVFR:
if useFFMPEG:
meta_r, frames_r = self.getVideoMeta(
source if inverse else target,
show_streams=True,
with_frames=True,
media_types=[expectedType])
index_r = ffmpeg_api.get_stream_indices_of_type(
meta_o, expectedType)[0]
adjustPositionsFFMPEG(meta_r[index_r], frames_r[index_r], hits)
else:
adjustPositions(
self.getNodeFile(source)
if inverse else self.getNodeFile(target), hits)
transfer_masks(video_masks,
new_mask_set,
dropRate,
frame_time_function=lambda x, y: y +
(1000.0 / targetRate),
frame_count_function=lambda x, y: y + 1)
return new_mask_set
def testWarp(self):
source = self.locateFile('tests/videos/sample1.mov')
target = 'sample1_ffr_ex.mov'
source_set = video_tools.getMaskSetForEntireVideo(video_tools.FileMetaDataLocator(source),
start_time='29', end_time='55')
target_set = video_tools.getMaskSetForEntireVideoForTuples(video_tools.FileMetaDataLocator(target),
start_time_tuple=(video_tools.get_start_time_from_segment(source_set[0]), 0),
end_time_tuple=(video_tools.get_end_time_from_segment(source_set[0]), 0))
print(source_set[0])
extractor = MetaDataExtractor(GraphProxy(source,target))
new_mask_set = extractor.warpMask(source_set, source, source)
print(new_mask_set[0])
self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(source_set[0]))
self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(source_set[0]))
self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(source_set[0]))
self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(source_set[0]))
self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(source_set[0]))
self._add_mask_files_to_kill(source_set)
new_mask_set = extractor.warpMask(source_set, source, target)
self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0]))
self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0]))
self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0]))
source_mask_set = extractor.warpMask(new_mask_set, source, target, inverse=True)
self.assertTrue(abs(video_tools.get_frames_from_segment(source_mask_set[0]) - video_tools.get_frames_from_segment(source_set[0])) < 2)
self.assertTrue(abs(video_tools.get_end_time_from_segment(source_mask_set[0]) - video_tools.get_end_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2)
self.assertTrue(abs(video_tools.get_rate_from_segment(source_mask_set[0]) - video_tools.get_rate_from_segment(source_set[0])) < 0.1)
self.assertTrue(abs(video_tools.get_start_frame_from_segment(source_mask_set[0]) - video_tools.get_start_frame_from_segment(source_set[0])) < 2)
self.assertTrue(
abs(video_tools.get_start_time_from_segment(source_mask_set[0]) - video_tools.get_start_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2)
new_mask_set = extractor.warpMask(source_set, source, target, useFFMPEG=True)
self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0]))
self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0]))
self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0]))
source_mask_set = extractor.warpMask(new_mask_set, source, target, inverse=True, useFFMPEG=True)
self.assertTrue(abs(video_tools.get_frames_from_segment(source_mask_set[0]) - video_tools.get_frames_from_segment(source_set[0])) < 2)
self.assertTrue(abs(video_tools.get_end_time_from_segment(source_mask_set[0]) - video_tools.get_end_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2)
self.assertTrue(abs(video_tools.get_rate_from_segment(source_mask_set[0]) - video_tools.get_rate_from_segment(source_set[0])) < 0.1)
self.assertTrue(abs(video_tools.get_start_frame_from_segment(source_mask_set[0]) - video_tools.get_start_frame_from_segment(source_set[0])) < 2)
self.assertTrue(
abs(video_tools.get_start_time_from_segment(source_mask_set[0]) - video_tools.get_start_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2)
source_set = target_set
source = target
target = 'sample1_ffr_2_ex.mov'
target_set = video_tools.getMaskSetForEntireVideoForTuples(video_tools.FileMetaDataLocator(target),
start_time_tuple=(video_tools.get_start_time_from_segment(source_set[0]), 0),
end_time_tuple=(video_tools.get_end_time_from_segment(source_set[0]), 0))
new_mask_set = extractor.warpMask(new_mask_set, source, target)
self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0]))
self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0]))
self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0]))
self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0]))