def test_Audio_to_Video(self): source = self.locateFile('tests/videos/sample1.mov') extractor = MetaDataExtractor(GraphProxy(source, 'b')) masks = [video_tools.create_segment(endframe= 2618367, rate= 44100, starttime=0.0, frames= 2618367, startframe=1, endtime=59373.424, type='audio')] newMasks = extractor.create_video_for_audio(source, masks=masks) self.assertTrue(len(newMasks) > len(masks)) self.assertTrue(video_tools.get_start_frame_from_segment(newMasks[1]) == 1) self.assertTrue(video_tools.get_end_frame_from_segment(newMasks[1]) == 803) self.assertTrue(video_tools.get_rate_from_segment(newMasks[1]) == 28.25) self.assertTrue(video_tools.get_end_time_from_segment(newMasks[1]) == 59348.333) source = self.locateFile('tests/videos/Sample1_slow.mov') masks = [video_tools.create_segment(endframe= 441000, rate= 44100, starttime=1000.0, frames= 396901, startframe=44100, endtime=10000.0, type='audio')] newMasks = extractor.create_video_for_audio(source, masks=masks) self.assertTrue(len(newMasks) > len(masks)) self.assertTrue(video_tools.get_rate_from_segment(newMasks[1]) == 10.0) self.assertTrue(video_tools.get_start_frame_from_segment(newMasks[1]) == 11) self.assertTrue(video_tools.get_end_frame_from_segment(newMasks[1]) == 100)
def run_warp(source, target, start_time, end_time): source_set = video_tools.FileMetaDataLocator( source).getMaskSetForEntireVideo(start_time=start_time, end_time=end_time) self.create_masks(source_set) extractor = MetaDataExtractor(GraphProxy(source, target)) target_set = video_tools.FileMetaDataLocator( target).getMaskSetForEntireVideoForTuples( start_time_tuple=(video_tools.get_start_time_from_segment( source_set[0]), 0), end_time_tuple=(video_tools.get_end_time_from_segment( source_set[0]), 0)) new_mask_set = extractor.warpMask(source_set, source, target) self.assertTrue( video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0])) self.assertTrue( video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0])) self.assertTrue( video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0])) self.assertTrue( video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0])) self.assertTrue( video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0])) file_data = self.read_masks(new_mask_set) self.assertEqual( video_tools.get_frames_from_segment(new_mask_set[0]), file_data[0]['frames'])
def testWarp(self): source = self.locateFile('tests/videos/sample1.mov') target = 'sample1_ffr_ex.mov' source_set = video_tools.getMaskSetForEntireVideo(video_tools.FileMetaDataLocator(source), start_time='29', end_time='55') target_set = video_tools.getMaskSetForEntireVideoForTuples(video_tools.FileMetaDataLocator(target), start_time_tuple=(video_tools.get_start_time_from_segment(source_set[0]), 0), end_time_tuple=(video_tools.get_end_time_from_segment(source_set[0]), 0)) print(source_set[0]) extractor = MetaDataExtractor(GraphProxy(source,target)) new_mask_set = extractor.warpMask(source_set, source, source) print(new_mask_set[0]) self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(source_set[0])) self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(source_set[0])) self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(source_set[0])) self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(source_set[0])) self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(source_set[0])) self._add_mask_files_to_kill(source_set) new_mask_set = extractor.warpMask(source_set, source, target) self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0])) self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0])) self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0])) source_mask_set = extractor.warpMask(new_mask_set, source, target, inverse=True) self.assertTrue(abs(video_tools.get_frames_from_segment(source_mask_set[0]) - video_tools.get_frames_from_segment(source_set[0])) < 2) self.assertTrue(abs(video_tools.get_end_time_from_segment(source_mask_set[0]) - video_tools.get_end_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2) self.assertTrue(abs(video_tools.get_rate_from_segment(source_mask_set[0]) - video_tools.get_rate_from_segment(source_set[0])) < 0.1) self.assertTrue(abs(video_tools.get_start_frame_from_segment(source_mask_set[0]) - video_tools.get_start_frame_from_segment(source_set[0])) < 2) self.assertTrue( abs(video_tools.get_start_time_from_segment(source_mask_set[0]) - video_tools.get_start_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2) new_mask_set = extractor.warpMask(source_set, source, target, useFFMPEG=True) self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0])) self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0])) self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0])) source_mask_set = extractor.warpMask(new_mask_set, source, target, inverse=True, useFFMPEG=True) self.assertTrue(abs(video_tools.get_frames_from_segment(source_mask_set[0]) - video_tools.get_frames_from_segment(source_set[0])) < 2) self.assertTrue(abs(video_tools.get_end_time_from_segment(source_mask_set[0]) - video_tools.get_end_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2) self.assertTrue(abs(video_tools.get_rate_from_segment(source_mask_set[0]) - video_tools.get_rate_from_segment(source_set[0])) < 0.1) self.assertTrue(abs(video_tools.get_start_frame_from_segment(source_mask_set[0]) - video_tools.get_start_frame_from_segment(source_set[0])) < 2) self.assertTrue( abs(video_tools.get_start_time_from_segment(source_mask_set[0]) - video_tools.get_start_time_from_segment(source_set[0])) < video_tools.get_error_from_segment(source_mask_set[0]) * 2) source_set = target_set source = target target = 'sample1_ffr_2_ex.mov' target_set = video_tools.getMaskSetForEntireVideoForTuples(video_tools.FileMetaDataLocator(target), start_time_tuple=(video_tools.get_start_time_from_segment(source_set[0]), 0), end_time_tuple=(video_tools.get_end_time_from_segment(source_set[0]), 0)) new_mask_set = extractor.warpMask(new_mask_set, source, target) self.assertTrue(video_tools.get_frames_from_segment(new_mask_set[0]) == video_tools.get_frames_from_segment(target_set[0])) self.assertTrue(video_tools.get_end_time_from_segment(new_mask_set[0]) == video_tools.get_end_time_from_segment(target_set[0])) self.assertTrue(video_tools.get_rate_from_segment(new_mask_set[0]) == video_tools.get_rate_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_frame_from_segment(new_mask_set[0]) == video_tools.get_start_frame_from_segment(target_set[0])) self.assertTrue(video_tools.get_start_time_from_segment(new_mask_set[0]) == video_tools.get_start_time_from_segment(target_set[0]))
def test_audio_zip_donor(self): graph = Mock() def lkup_preds(x): return {'b': ['a'], 'e': ['d']}[x] def lkup_edge(x, y): return \ {'ab': {'op': 'NoSelect'}, 'ef': {'op': 'SelectSomething', 'arguments': {'Start Time': "00:00:00.000000"}}}[ x + y] graph.predecessors = lkup_preds graph.get_edge = lkup_edge graph.dir = '.' donor = AudioZipDonor( graph, 'e', 'f', 'x', (None, self.locateFile('tests/zips/test.wav.zip')), (None, self.locateFile('tests/videos/sample1.mov'))) args = donor.arguments() self.assertEqual("00:00:00.000000", args['Start Time']['defaultvalue']) segments = donor.create( arguments={ 'Start Time': "00:00:09.11", 'End Time': "00:00:16.32", 'sample rate': 44100 }) for segment in segments: self.assertEqual(401752, get_start_frame_from_segment(segment)) self.assertEqual(719713, get_end_frame_from_segment(segment)) self.assertAlmostEqual(9110, get_start_time_from_segment(segment), places=1) self.assertEqual(16320.0, int(get_end_time_from_segment(segment))) segments = donor.create( arguments={ 'Start Time': "00:00:00.00", 'End Time': "00:00:00.00", 'sample rate': 44100 }) for segment in segments: self.assertEqual(1, get_start_frame_from_segment(segment)) self.assertEqual(1572865, get_end_frame_from_segment(segment)) self.assertAlmostEqual(0.0, get_start_time_from_segment(segment), places=1) self.assertEqual(35665, int(get_end_time_from_segment(segment)))
def create_masks(self, mask_set): from maskgen import tool_set import numpy as np for segment in mask_set: writer = tool_set.GrayBlockWriter( 'test_warp', video_tools.get_rate_from_segment(segment)) for i in range(video_tools.get_frames_from_segment(segment)): f = i + video_tools.get_start_frame_from_segment(segment) writer.write( np.random.randint(0, 1, (1000, 1000), 'uint8') * 255, (f - 1) * video_tools.get_rate_from_segment(segment), frame_number=f) writer.close() video_tools.update_segment(segment, videosegment=writer.filename) self.addFileToRemove(writer.filename)
def test_link_subsitite_masks(self): import os def imageName(x): return None, x createMovie( 'test_link1.m4v', lambda: np.random.randint(0, 255, (720, 480, 3), dtype='uint8'), 40) self.addFileToRemove('test_link1.m4v') createMovie( 'test_link2.m4v', lambda: np.random.randint(0, 255, (720, 480, 3), dtype='uint8'), 40) self.addFileToRemove('test_link2.m4v') masks = video_tools.formMaskDiff('test_link1.m4v', 'test_link2.m4v', 'link_1_2_cmp', '-') createMovie( 'test_link3.m4v', lambda: np.random.randint(0, 255, (720, 480, 3), dtype='uint8'), 30) self.addFileToRemove('test_link3.m4v') link_tool = scenario_model.VideoVideoLinkTool() model = Mock() model.getImageAndName = imageName model.G = Mock() model.G.dir = '.' subs = link_tool.addSubstituteMasks('test_link1.m4v', 'test_link2.m4v', model, '-', arguments={'Start Time': '11'}, filename='test_link3.m4v') self.assertEqual(1, len(subs)) self.assertEqual(30, video_tools.get_frames_from_segment(subs[0])) self.assertEqual(11, video_tools.get_start_frame_from_segment(subs[0])) self.assertTrue( os.path.exists(video_tools.get_file_from_segment(subs[0]))) self.addFileToRemove(video_tools.get_file_from_segment(subs[0]))
def test_audio_donor(self): graph = Mock() def lkup_preds(x): return {'b': ['a'], 'e': ['d']}[x] def lkup_edge(x, y): return \ {'ab': {'op': 'NoSelect'}, 'de': {'op': 'SelectSomething', 'arguments': {'Start Time': 20, 'End Time': 100}}}[ x + y] graph.predecessors = lkup_preds graph.get_edge = lkup_edge graph.dir = '.' donor = AudioDonor(graph, 'e', 'f', 'x', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) args = donor.arguments() self.assertEqual("00:00:00.000000", args['Start Time']['defaultvalue']) self.assertEqual("00:00:00.000000", args['End Time']['defaultvalue']) segments = donor.create(arguments={ 'Start Time': "00:00:01.11", 'End Time': "00:00:01.32" }) for segment in segments: self.assertEqual(48951, get_start_frame_from_segment(segment)) self.assertEqual(58212, get_end_frame_from_segment(segment)) self.assertAlmostEqual(1109.97, get_start_time_from_segment(segment), places=1) self.assertEqual(1320.0, int(get_end_time_from_segment(segment))) donor = AllStreamDonor( graph, 'e', 'f', 'y', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) args = donor.arguments() self.assertEqual(0, len(args)) segments = donor.create(arguments={}) types = set() for segment in segments: types.add(get_type_of_segment(segment)) if get_type_of_segment(segment) == 'audio': self.assertEqual(1, get_start_frame_from_segment(segment)) self.assertEqual(2617262, get_end_frame_from_segment(segment)) self.assertAlmostEqual(0, get_start_time_from_segment(segment), places=1) self.assertAlmostEqual(59348, int(get_end_time_from_segment(segment))) else: self.assertEqual(1, get_start_frame_from_segment(segment)) self.assertEqual(803, get_end_frame_from_segment(segment)) self.assertAlmostEqual(0, get_start_time_from_segment(segment), places=1) self.assertAlmostEqual(59348, int(get_end_time_from_segment(segment))) self.assertEqual(2, len(types)) donor = AllAudioStreamDonor( graph, 'e', 'f', 'y', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) self.assertEqual(0, len(donor.arguments())) self.assertEqual(['audio'], donor.media_types())
def test_video_donor(self): graph = Mock() def lkup_preds(x): return {'b': ['a'], 'e': ['d']}[x] def lkup_edge(x, y): return { 'ab': { 'op': 'NoSelect' }, 'de': { 'op': 'SelectSomething', 'arguments': { 'Start Time': 20, 'End Time': 100 } } }[x + y] graph.predecessors = lkup_preds graph.get_edge = lkup_edge graph.dir = '.' donor = VideoDonor(graph, 'e', 'f', 'x', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) args = donor.arguments() self.assertEqual(20, args['Start Time']['defaultvalue']) self.assertEqual(100, args['End Time']['defaultvalue']) segments = donor.create(arguments={ 'include audio': 'yes', 'Start Time': 30, 'End Time': 150 }) for segment in segments: if get_type_of_segment(segment) == 'audio': self.assertEqual(115542, get_start_frame_from_segment(segment)) self.assertEqual(509061, get_end_frame_from_segment(segment)) else: self.assertEqual(30, get_start_frame_from_segment(segment)) self.assertEqual(150, get_end_frame_from_segment(segment)) self.assertEqual(2620.0, get_start_time_from_segment(segment)) self.assertEqual(11543, int(get_end_time_from_segment(segment))) donor = VideoDonor(graph, 'b', 'c', 'x', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) args = donor.arguments() self.assertEqual(1, args['Start Time']['defaultvalue']) self.assertEqual(0, args['End Time']['defaultvalue']) segments = donor.create(arguments={ 'include audio': 'yes', 'Start Time': 30, 'End Time': 150 }) for segment in segments: if get_type_of_segment(segment) == 'audio': self.assertEqual(115542, get_start_frame_from_segment(segment)) self.assertEqual(509061, get_end_frame_from_segment(segment)) else: self.assertEqual(30, get_start_frame_from_segment(segment)) self.assertEqual(150, get_end_frame_from_segment(segment)) self.assertEqual(2620.0, get_start_time_from_segment(segment)) self.assertEqual(11543, int(get_end_time_from_segment(segment))) segments = donor.create(arguments={ 'include audio': 'no', 'Start Time': 30, 'End Time': 150 }) self.assertEqual( 0, len([ segment for segment in segments if get_type_of_segment(segment) == 'audio' ])) donor = VideoDonorWithoutAudio( graph, 'b', 'c', 'x', (None, self.locateFile('tests/videos/sample1.mov')), (None, self.locateFile('tests/videos/sample1.mov'))) self.assertTrue('include audio' not in donor.arguments())
def warpMask(self, video_masks, source, target, expectedType='video', inverse=False, useFFMPEG=False): """ Tranform masks when the frame rate has changed. :param video_masks: ithe set of video masks to walk through and transform :param expectedType: :param video_masks: :return: new set of video masks """ edge = self.graph.get_edge(source, target) meta_i, frames_i = self.getVideoMeta(source, show_streams=True, media_types=[expectedType]) meta_o, frames_o = self.getVideoMeta(target, show_streams=True, media_types=[expectedType]) indices_i = ffmpeg_api.get_stream_indices_of_type(meta_i, expectedType) indices_o = ffmpeg_api.get_stream_indices_of_type(meta_o, expectedType) if not indices_i or not indices_o: return video_masks index_i = indices_i[0] index_o = indices_o[0] isVFR = ffmpeg_api.is_vfr(meta_i[index_i]) or ffmpeg_api.is_vfr( meta_o[index_o]) result = self.getChangeInFrames(edge, meta_i[index_i], meta_o[index_o], source, target, expectedType=expectedType) if result is None: return video_masks sourceFrames, sourceTime, targetFrames, targetTime, sourceRate, targetRate = result if sourceFrames == targetFrames and int(sourceTime * 100) == int( targetTime * 100): return video_masks dropRate = sourceFrames / float( sourceFrames - targetFrames) if sourceFrames != targetFrames else sourceFrames + 1 #if sourceFrames > targetFrames else targetFrames / float(sourceFrames - targetFrames) def apply_change(existing_value, orig_rate, final_rate, inverse=False, round_value=True, min_value=0, upper_bound=False): # if round_value, return a tuple of value plus rounding error import math multiplier = -1.0 if inverse else 1.0 adjustment = existing_value * math.pow(final_rate / orig_rate, multiplier) if round_value: v = max( min(round(adjustment), final_rate) if upper_bound else round(adjustment), min_value) e = abs(adjustment - v) return int(v), e return max( min(adjustment, final_rate) if upper_bound else adjustment, min_value) def adjustPositionsFFMPEG(meta, video_frames, hits): rate = ffmpeg_api.get_video_frame_rate_from_meta([meta], [video_frames]) aptime = 0 lasttime = 0 hitspos = 0 start_mask = None for pos in range(0, len(video_frames)): aptime = get_frame_time(video_frames[pos], aptime, rate) while hitspos < len(hits) and aptime > hits[hitspos][0]: mask = hits[hitspos][2] element = hits[hitspos][1] error = abs(aptime - hits[hitspos][0]) if element == 'starttime': update_segment(mask, starttime=lasttime, startframe=pos, error=error + get_error_from_segment(mask)) start_mask = mask else: # for error, only record the error if not recorded update_segment( mask, endtime=lasttime, endframe=pos, error=(error if start_mask != mask else 0) + get_error_from_segment(mask)) hitspos += 1 lasttime = aptime return mask def adjustPositions(video_file, hits): # used if variable frame rate frmcnt = 0 hitspos = 0 last = 0 cap = cv2api_delegate.videoCapture(video_file) try: while cap.grab() and hitspos < len(hits): frmcnt += 1 aptime = cap.get(cv2api_delegate.prop_pos_msec) while hitspos < len(hits) and aptime > hits[hitspos][0]: mask = hits[hitspos][2] element = hits[hitspos][1] error = max(abs(last - hits[hitspos][0]), abs(aptime - hits[hitspos][0])) if element == 'starttime': update_segment(mask, starttime=last, startframe=frmcnt, error=error) else: update_segment(mask, endtime=last, endframe=frmcnt, error=max( error, get_error_from_segment(mask))) hitspos += 1 last = aptime finally: cap.release() return mask new_mask_set = [] hits = [] # First adjust all the frame and time references by the total change in the video. # In most cases, the length of the video in time changes by a small amount which is distributed # across all the masks for mask_set in video_masks: if 'type' in mask_set and mask_set['type'] != expectedType: new_mask_set.append(mask_set) continue #these are initial estimates startframe, error_start = apply_change( get_start_frame_from_segment(mask_set), float(sourceFrames), float(targetFrames), inverse=inverse, round_value=True, min_value=1) endframe, error_end = apply_change( get_end_frame_from_segment(mask_set), float(sourceFrames), float(targetFrames), inverse=inverse, min_value=1, round_value=True, upper_bound=True) endtime = apply_change(get_end_time_from_segment(mask_set), float(sourceTime), targetTime, inverse=inverse, round_value=False) starttime = apply_change(get_start_time_from_segment(mask_set), sourceTime, targetTime, inverse=inverse, round_value=False, upper_bound=True) try: if endframe == int(getValue(meta_o[index_o], 'nb_frames', 0)) and \ float(getValue(meta_o[index_o], 'duration', 0)) > 0: endtime = float(getValue(meta_o[index_o], 'duration', 0)) * 1000.0 - (1000.0 / targetRate) elif endtime > targetTime and endframe > targetFrames: message = '{} exceeded target time of {} for {}'.format( sourceTime, target, targetTime) if (endtime - targetTime) > 300: logging.getLogger('maskgen').error(message) else: logging.getLogger('maskgen').warn(message) endtime = targetTime - (1000.0 / targetRate) endframe = targetFrames except: pass change = create_segment( rate=sourceRate if inverse else targetRate, type=get_type_of_segment(mask_set), starttime=starttime, startframe=startframe, error=get_error_from_segment(mask_set) + (max(error_start, error_end) / targetRate * 1000.0), endtime=endtime, endframe=endframe, videosegment=get_file_from_segment(mask_set)) if not isVFR: # in this case, we trust the time invariance, updating frames recalculate_frames_for_segment(change) # then we reupdate time to match the frames recalculate_times_for_segment(change) new_mask_set.append(change) hits.append( (get_start_time_from_segment(change), 'starttime', change)) hits.append((get_end_time_from_segment(change), 'endime', change)) # only required when one of the two videos is variable rate hits = sorted(hits) if isVFR: if useFFMPEG: meta_r, frames_r = self.getVideoMeta( source if inverse else target, show_streams=True, with_frames=True, media_types=[expectedType]) index_r = ffmpeg_api.get_stream_indices_of_type( meta_o, expectedType)[0] adjustPositionsFFMPEG(meta_r[index_r], frames_r[index_r], hits) else: adjustPositions( self.getNodeFile(source) if inverse else self.getNodeFile(target), hits) transfer_masks(video_masks, new_mask_set, dropRate, frame_time_function=lambda x, y: y + (1000.0 / targetRate), frame_count_function=lambda x, y: y + 1) return new_mask_set