def test_video_frame_stim(): filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename, onset=4.2) frame = VideoFrameStim(video, 42) assert frame.onset == (5.6) assert np.array_equal(frame.data, video.get_frame(index=42).data) assert frame.name == 'frame[42]'
def test_frame_sampling_video_filter2(): filename = join(VIDEO_DIR, 'obama_speech.mp4') video = VideoStim(filename, onset=4.2) assert video.fps == 12 assert video.n_frames == 105 # Test frame indices conv = FrameSamplingFilter(every=3) derived = conv.transform(video) assert derived.n_frames == 35 assert derived.frame_index[4] == 12 conv = FrameSamplingFilter(hertz=3) derived = conv.transform(video) assert derived.n_frames == 27 assert derived.frame_index[3] == 12 conv = FrameSamplingFilter(hertz=24) derived = conv.transform(video) assert derived.n_frames == 210 assert derived.frame_index[4] == 2 video.fps = 11.8 conv = FrameSamplingFilter(hertz=1) derived = conv.transform(video) assert derived.n_frames == 9 assert derived.frame_index[4] == 47 assert derived.frame_index[5] == 59
def test_google_vision_face_batch(): obama_file = join(get_test_data_path(), 'image', 'obama.jpg') people_file = join(get_test_data_path(), 'image', 'thai_people.jpg') stims = [ImageStim(obama_file), ImageStim(people_file)] ext = GoogleVisionAPIFaceExtractor(handle_annotations='first') result = ext.transform(stims) result = ExtractorResult.merge_stims(result) assert 'face1_joyLikelihood' in result.columns assert result['face1_joyLikelihood'][0] == 'VERY_LIKELY' assert result['face1_joyLikelihood'][1] == 'VERY_LIKELY' video = VideoStim(join(get_test_data_path(), 'video', 'obama_speech.mp4')) conv = FrameSamplingFilter(every=10) video = conv.transform(video) result = ext.transform(video) result = ExtractorResult.merge_stims(result) assert 'face1_joyLikelihood' in result.columns assert result.shape == (11, 137) video = VideoStim(join(get_test_data_path(), 'video', 'small.mp4')) video = conv.transform(video) result = ext.transform(video) result = ExtractorResult.merge_stims(result) assert 'face1_joyLikelihood' not in result.columns assert result.shape == (17, 7)
def test_clarifai_api_video_extractor(): stim = VideoStim(join(VIDEO_DIR, 'small.mp4')) ext = ClarifaiAPIVideoExtractor() assert ext.validate_keys() result = ext.transform(stim).to_df() # This should actually be 6, in principle, because the clip is < 6 seconds, # but the Clarifai API is doing weird things. See comment in # ClarifaiAPIVideoExtractor._to_df() for further explanation. assert result.shape[0] in (6, 7) # Changes sometimes, so use a range assert result.shape[1] > 25 and result.shape[1] < 30 assert result['toy'][0] > 0.5 assert result['onset'][1] == 1.0 assert result['duration'][0] == 1.0 # because of the behavior described above—handle both cases assert np.isclose(result['duration'][5], 0.57) or result['duration'][6] == 0 ext = ClarifaiAPIVideoExtractor(model='face') result = ext.transform(stim).to_df() keys_to_check = ['top_row', 'left_col', 'bottom_row', 'right_col'] assert [k not in result.keys() for k in keys_to_check] assert all([result[k][0] == np.nan for k in result if k in keys_to_check]) stim = VideoStim(join(VIDEO_DIR, 'obama_speech.mp4')) result = ext.transform(stim).to_df() keys_to_check = ['top_row', 'left_col', 'bottom_row', 'right_col'] assert [k in result.keys() for k in keys_to_check]
def test_google_video_api_shot_extractor(caplog): ext = GoogleVideoAPIShotDetectionExtractor(request_rate=3) stim = VideoStim(join(VIDEO_DIR, 'small.mp4')) result = ext.transform(stim).to_df() log_message = caplog.records[-1].message incomplete = (log_message == ("The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 90)) if not incomplete: assert result.shape == (1, 5) assert result['onset'][0] == 0.0 assert np.isclose(result['duration'][0], stim.duration, 0.1) assert 'shot_id' in result.columns assert result['shot_id'][0] == 0 ext = GoogleVideoAPIShotDetectionExtractor() stim = VideoStim(join(VIDEO_DIR, 'shot_change.mp4')) result = ext.transform(stim).to_df() log_message = caplog.records[-1].message incomplete = (log_message == ("The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 90)) if not incomplete: assert result.shape == (2, 5) assert np.isclose(result['onset'][1], 3.2, 0.1) assert 'shot_id' in result.columns assert result['shot_id'][1] == 1
def test_google_vision_face_batch(): stims = ['apple', 'obama', 'thai_people'] stim_files = [join(get_test_data_path(), 'image', '%s.jpg' % s) for s in stims] stims = [ImageStim(s) for s in stim_files] ext = GoogleVisionAPIFaceExtractor(batch_size=5) result = ext.transform(stims) result = merge_results(result, format='wide', extractor_names=False, handle_annotations='first') assert result.shape == (2, 139) assert 'joyLikelihood' in result.columns assert result['joyLikelihood'][0] == 'VERY_LIKELY' assert result['joyLikelihood'][1] == 'VERY_LIKELY' video = VideoStim(join(VIDEO_DIR, 'obama_speech.mp4')) conv = FrameSamplingFilter(every=10) video = conv.transform(video) result = ext.transform(video) result = merge_results(result, format='wide', extractor_names=False) assert 'joyLikelihood' in result.columns assert result.shape == (22, 139) video = VideoStim(join(VIDEO_DIR, 'small.mp4')) video = conv.transform(video) result = ext.transform(video) result = merge_results(result, format='wide', extractor_names=False) assert 'joyLikelihood' not in result.columns assert len(result) == 0
def test_video_stim_bytestring(): path = join(get_test_data_path(), 'video', 'small.mp4') vid = VideoStim(path) assert vid._bytestring is None bs = vid.get_bytestring() assert isinstance(bs, str) assert vid._bytestring is not None raw = bs.encode() with open(path, 'rb') as f: assert raw == base64.b64encode(f.read())
def test_frame_sampling_video_filter(): filename = join(VIDEO_DIR, 'small.mp4') video = VideoStim(filename, onset=4.2) assert video.fps == 30 assert video.n_frames in (167, 168) assert video.width == 560 # Test frame filters conv = FrameSamplingFilter(every=3) derived = conv.transform(video) assert derived.n_frames == math.ceil(video.n_frames / 3.0) assert derived.duration == video.duration first = next(f for f in derived) assert type(first) == VideoFrameStim assert first.name == 'frame[0]' assert first.onset == 4.2 assert first.duration == 3 * (1 / 30.0) second = [f for f in derived][1] assert second.onset == 4.3 with pytest.raises(TypeError): derived.get_frame(onset=1.0) # Commented out because no longer allowing sampling filter chaining # conv = FrameSamplingFilter(hertz=15) # derived = conv.transform(derived) # assert derived.n_frames == math.ceil(video.n_frames / 6.0) # first = next(f for f in derived) # assert type(first) == VideoFrameStim # assert first.duration == 3 * (1 / 15.0) # second = [f for f in derived][1] # assert second.onset == 4.4 with pytest.raises(TypeError): conv.transform(derived)
def resample_video(video_file, sampling_rate): """ This function resamples a video to the desired sampling rate. Can be useful for making video with high sampling rates more tractable for analysis. Parameters ---------- video_file: str file path to video to be resampled. sampling_rate: float Desired sampling rate in Hz Returns ------- resampled_video: pliers video object with resampled video frames """ from pliers.stimuli import VideoStim from pliers.filters import FrameSamplingFilter video = VideoStim(video_file) resamp_filter = FrameSamplingFilter(hertz=sampling_rate) resampled_video = resamp_filter.transform(video) return resampled_video
def test_compound_stim(): audio_dir = join(get_test_data_path(), 'audio') audio = AudioStim(join(audio_dir, 'crowd.mp3')) image1 = ImageStim(join(get_test_data_path(), 'image', 'apple.jpg')) image2 = ImageStim(join(get_test_data_path(), 'image', 'obama.jpg')) filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename) text = ComplexTextStim(text="The quick brown fox jumped...") stim = CompoundStim([audio, image1, image2, video, text]) assert len(stim.elements) == 5 assert isinstance(stim.video, VideoStim) assert isinstance(stim.complex_text, ComplexTextStim) assert isinstance(stim.image, ImageStim) with pytest.raises(AttributeError): stim.nonexistent_type assert stim.video_frame is None # Test iteration len([e for e in stim]) == 5 imgs = stim.get_stim(ImageStim, return_all=True) assert len(imgs) == 2 assert all([isinstance(im, ImageStim) for im in imgs]) also_imgs = stim.get_stim('image', return_all=True) assert imgs == also_imgs
def test_big_pipeline(): pytest.importorskip('pygraphviz') filename = join(get_test_data_path(), 'video', 'obama_speech.mp4') video = VideoStim(filename) visual_nodes = [(FrameSamplingFilter(every=15), [ (TesseractConverter(), [LengthExtractor()]), VibranceExtractor(), 'BrightnessExtractor', ])] audio_nodes = [(VideoToAudioConverter(), [WitTranscriptionConverter(), 'LengthExtractor'], 'video_to_audio')] graph = Graph() graph.add_nodes(visual_nodes) graph.add_nodes(audio_nodes) results = graph.run(video, merge=False) result = merge_results(results, format='wide', extractor_names='multi') # Test that pygraphviz outputs a file drawfile = next(tempfile._get_candidate_names()) graph.draw(drawfile) assert exists(drawfile) os.remove(drawfile) assert ('LengthExtractor', 'text_length') in result.columns assert ('VibranceExtractor', 'vibrance') in result.columns # assert not result[('onset', '')].isnull().any() assert 'text[negotiations]' in result['stim_name'].values assert 'frame[90]' in result['stim_name'].values
def test_optical_flow_extractor(): pytest.importorskip('cv2') stim = VideoStim(join(VIDEO_DIR, 'small.mp4'), onset=4.2) result = FarnebackOpticalFlowExtractor().transform(stim).to_df() target = result.query('onset==7.2')['total_flow'] # Value returned by cv2 seems to change over versions, so use low precision assert np.isclose(target, 86248.05, 1e-4)
def test_google_video_api_extractor(caplog): ext = GoogleVideoIntelligenceAPIExtractor(timeout=1) stim = VideoStim(join(VIDEO_DIR, 'park.mp4')) result = ext.transform(stim) log_message = caplog.records[-1].message assert log_message == ("The extraction reached the timeout limit of %fs, " "which means the API may not have finished analyzing the " "video and the results may be empty or incomplete." % 1.0) ext = GoogleVideoIntelligenceAPIExtractor(timeout=500, features=['LABEL_DETECTION', 'SHOT_CHANGE_DETECTION']) result = ext.transform(stim).to_df() log_message = caplog.records[-1].message incomplete = (log_message == ("The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 500)) if not incomplete: assert result.shape == (1, 31) assert result['onset'][0] == 0.0 assert result['duration'][0] > 0.5 and result['duration'][0] < 0.6 assert result['category_plant'][0] > 0.5 assert result['park'][0] > 0.5 assert result['shot_id'][0] == 0
def test_derived_video_converter(): filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename, onset=4.2) assert video.fps == 30 assert video.n_frames in (167, 168) assert video.width == 560 # Test frame filters conv = FrameSamplingConverter(every=3) derived = conv.transform(video) assert len(derived._frames) == math.ceil(video.n_frames / 3.0) first = next(f for f in derived) assert type(first) == VideoFrameStim assert first.name == 'frame[0]' assert first.onset == 4.2 assert first.duration == 3 * (1 / 30.0) second = [f for f in derived][1] assert second.onset == 4.3 # Should refilter from original frames conv = FrameSamplingConverter(hertz=15) derived = conv.transform(derived) assert len(derived._frames) == math.ceil(video.n_frames / 6.0) first = next(f for f in derived) assert type(first) == VideoFrameStim assert first.duration == 3 * (1 / 15.0) second = [f for f in derived][1] assert second.onset == 4.4
def test_optical_flow_extractor(): pytest.importorskip('cv2') video_dir = join(get_test_data_path(), 'video') stim = VideoStim(join(video_dir, 'small.mp4')) result = DenseOpticalFlowExtractor().transform(stim).to_df() target = result.query('onset==3.0')['total_flow'] # Value returned by cv2 seems to change over versions, so use low precision assert np.isclose(target, 86248.05, 1e-4)
def test_multistep_converter(): conv = VideoToTextConverter() filename = join(get_test_data_path(), 'video', 'obama_speech.mp4') stim = VideoStim(filename) text = conv.transform(stim) assert isinstance(text, ComplexTextStim) first_word = next(w for w in text) assert type(first_word) == TextStim
def test_video_to_audio_converter(): filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename) conv = VideoToAudioConverter() audio = conv.transform(video) assert audio.history.source_class == 'VideoStim' assert audio.history.source_file == filename assert np.isclose(video.duration, audio.duration, 1e-2)
def test_implicit_stim_conversion3(): video_dir = join(get_test_data_path(), 'video') stim = VideoStim(join(video_dir, 'obama_speech.mp4')) ext = LengthExtractor() result = ext.transform(stim) first_word = result[0].to_df() # The word should be "today" assert 'text_length' in first_word.columns assert first_word['text_length'][0] == 5
def test_video_to_audio_converter(): filename = join(VIDEO_DIR, 'small.mp4') video = VideoStim(filename, onset=4.2) conv = VideoToAudioConverter() audio = conv.transform(video) assert audio.history.source_class == 'VideoStim' assert audio.history.source_file == filename assert audio.onset == 4.2 assert np.isclose(video.duration, audio.duration, 1e-2)
def test_frame_sampling_cv2(): pytest.importorskip('cv2') filename = join(VIDEO_DIR, 'small.mp4') video = VideoStim(filename) conv = FrameSamplingFilter(top_n=5) derived = conv.transform(video) assert derived.n_frames == 5 assert type(next(f for f in derived)) == VideoFrameStim
def test_derived_video_converter_cv2(): pytest.importorskip('cv2') filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename) conv = FrameSamplingConverter(top_n=5) derived = conv.transform(video) assert len(derived._frames) == 5 assert type(next(f for f in derived)) == VideoFrameStim
def test_clarifai_api_video_extractor(): stim = VideoStim(join(VIDEO_DIR, 'small.mp4')) ext = ClarifaiAPIVideoExtractor() assert ext.validate_keys() result = ext.transform(stim).to_df() assert result.shape == (6, 27) assert result['toy'][0] > 0.5 assert result['onset'][1] == 1.0 assert result['duration'][0] == 1.0 assert np.isclose(result['duration'][5], 0.57)
def test_video_stim(): ''' Test VideoStim functionality. ''' filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename, onset=4.2) assert video.fps == 30 assert video.n_frames == 168 assert video.width == 560 assert video.duration == 5.57 # Test frame iterator frames = [f for f in video] assert len(frames) == 168 f1 = frames[100] assert isinstance(f1, VideoFrameStim) assert isinstance(f1.onset, float) assert np.isclose(f1.duration, 1 / 30.0, 1e-5) f1.data.shape == (320, 560, 3) # Test getting of specific frame f2 = video.get_frame(index=100) assert isinstance(f2, VideoFrameStim) assert isinstance(f2.onset, float) assert f2.onset > 7.5 f2.data.shape == (320, 560, 3) f2_copy = video.get_frame(onset=3.33334) assert isinstance(f2, VideoFrameStim) assert isinstance(f2.onset, float) assert f2.onset > 7.5 assert np.array_equal(f2.data, f2_copy.data) # Try another video filename = join(get_test_data_path(), 'video', 'obama_speech.mp4') video = VideoStim(filename) assert video.fps == 12 assert video.n_frames == 105 assert video.width == 320 assert video.duration == 8.71 f3 = video.get_frame(index=104) assert isinstance(f3, VideoFrameStim) assert isinstance(f3.onset, float) assert f3.duration > 0.0 assert f3.data.shape == (240, 320, 3)
def test_stim_history_tracking(): video = VideoStim(join(get_test_data_path(), 'video', 'obama_speech.mp4')) assert video.history is None conv = VideoToAudioConverter() stim = conv.transform(video) assert str(stim.history) == 'VideoStim->VideoToAudioConverter/AudioStim' conv = WitTranscriptionConverter() stim = conv.transform(stim) assert str( stim.history ) == 'VideoStim->VideoToAudioConverter/AudioStim->WitTranscriptionConverter/ComplexTextStim'
def test_video_stim(): ''' Test VideoStim functionality. ''' filename = join(get_test_data_path(), 'video', 'small.mp4') video = VideoStim(filename) assert video.fps == 30 assert video.n_frames in (167, 168) assert video.width == 560 # Test frame iterator frames = [f for f in video] assert len(frames) == 168 f1 = frames[100] assert isinstance(f1, VideoFrameStim) assert isinstance(f1.onset, float) f1.data.shape == (320, 560, 3) # Test getting of specific frame f2 = video.get_frame(index=100) assert isinstance(f2, VideoFrameStim) assert isinstance(f2.onset, float) f2.data.shape == (320, 560, 3)
def test_google_video_api_label_extractor(caplog): ext = GoogleVideoAPILabelDetectionExtractor(mode='FRAME_MODE', stationary_camera=True) stim = VideoStim(join(VIDEO_DIR, 'small.mp4')) ex_result = ext.transform(stim) log_message = caplog.records[-1].message incomplete = (log_message == ( "The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 90)) if not incomplete: result = ex_result.to_df() assert result.shape[1] > 20 assert 'category_toy' in result.columns assert result['toy'][0] > 0.5 assert np.isclose(result['duration'][0], stim.duration, 0.1) result = ex_result.to_df(format='long') assert 'pornographyLikelihood' not in result['feature'] assert np.nan not in result['value'] ext = GoogleVideoAPILabelDetectionExtractor(mode='SHOT_MODE') stim = VideoStim(join(VIDEO_DIR, 'shot_change.mp4')) ex_result = ext.transform(stim) log_message = caplog.records[-1].message incomplete = (log_message == ( "The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 90)) if not incomplete: raw = ex_result.raw['response']['annotationResults'][0] assert 'shotLabelAnnotations' in raw result = ex_result.to_df() assert result.shape[1] > 10 assert result['onset'][1] == 0.0 assert np.isclose(result['onset'][2], 3.2, 0.1) assert np.isnan(result['cat'][1]) assert result['cat'][2] > 0.5 assert np.isnan(result['clock'][2]) assert result['clock'][1] > 0.5 or result['clock'][0] > 0.5
def test_google_video_api_explicit_extractor(caplog): ext = GoogleVideoAPIExplicitDetectionExtractor(request_rate=3) stim = VideoStim(join(VIDEO_DIR, 'small.mp4'), onset=4.2) result = ext.transform(stim).to_df() log_message = caplog.records[-1].message incomplete = (log_message == ("The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 90)) if not incomplete: assert result.shape[1] == 5 assert result['onset'][0] >= 4.2 assert 'pornographyLikelihood' in result.columns assert 'UNLIKELY' in result['pornographyLikelihood'][0]
def test_save(): text_dir = join(get_test_data_path(), 'text') complextext_stim = ComplexTextStim(join(text_dir, 'complex_stim_no_header.txt'), columns='ot', default_duration=0.2) text_stim = TextStim(text='hello') video_stim = VideoStim(join(get_test_data_path(), 'video', 'small.mp4')) audio_stim = AudioStim(join(get_test_data_path(), 'audio', 'crowd.mp3')) image_stim = ImageStim(join(get_test_data_path(), 'image', 'apple.jpg')) stims = [complextext_stim, text_stim, video_stim, audio_stim, image_stim] for s in stims: path = tempfile.mktemp() + s._default_file_extension s.save(path) assert exists(path) os.remove(path)
def test_microsoft_vision_api_extractor_large(): default = config.get_option('allow_large_jobs') default_large = config.get_option('large_job') config.set_option('allow_large_jobs', False) config.set_option('large_job', 3) ext = MicrosoftVisionAPITagExtractor() video = VideoStim(join(VIDEO_DIR, 'small.mp4')) with pytest.raises(ValueError): merge_results(ext.transform(video)) config.set_option('allow_large_jobs', default) config.set_option('large_job', default_large)
def test_video_trimming_filter(): video = VideoStim(join(VIDEO_DIR, 'small.mp4')) filt = TemporalTrimmingFilter(end=4.0) short_video = filt.transform(video) assert short_video.fps == 30 assert short_video.duration == 4.0 frame_filt = VideoTrimmingFilter(end=100, frames=True) short_video = frame_filt.transform(video) assert short_video.fps == 30 assert short_video.n_frames == 100 error_filt = VideoTrimmingFilter(end=10.0, validation='strict') with pytest.raises(ValueError): short_video = error_filt.transform(video)
def test_google_video_api_extractor2(caplog): segments = [{'startTimeOffset': '0.1s', 'endTimeOffset': '0.3s'}, {'startTimeOffset': '0.3s', 'endTimeOffset': '0.45s'}] ext = GoogleVideoIntelligenceAPIExtractor(timeout=500, segments=segments, features=['EXPLICIT_CONTENT_DETECTION']) stim = VideoStim(join(VIDEO_DIR, 'park.mp4')) result = ext.transform(stim).to_df() log_message = caplog.records[-1].message incomplete = (log_message == ("The extraction reached the timeout limit of" " %fs, which means the API may not have finished analyzing the" " video and the results may be empty or incomplete." % 500)) if not incomplete: assert result.shape == (2, 5) assert result['onset'][0] > 0.1 and result['onset'][0] < 0.3 assert result['onset'][1] > 0.3 and result['onset'][1] < 0.45 assert 'UNLIKELY' in result['pornographyLikelihood'][0]