def test_clarifai_api_video_extractor():
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'))
ext = ClarifaiAPIVideoExtractor()
assert ext.validate_keys()
result = ext.transform(stim).to_df()
# This should actually be 6, in principle, because the clip is < 6 seconds,
# but the Clarifai API is doing weird things. See comment in
# ClarifaiAPIVideoExtractor._to_df() for further explanation.
assert result.shape[0] in (6, 7)
# Changes sometimes, so use a range
assert result.shape[1] > 25 and result.shape[1] < 30
assert result['toy'][0] > 0.5
assert result['onset'][1] == 1.0
assert result['duration'][0] == 1.0
# because of the behavior described above—handle both cases
assert np.isclose(result['duration'][5],
0.57) or result['duration'][6] == 0
ext = ClarifaiAPIVideoExtractor(model='face')
result = ext.transform(stim).to_df()
keys_to_check = ['top_row', 'left_col', 'bottom_row', 'right_col']
assert [k not in result.keys() for k in keys_to_check]
assert all([result[k][0] == np.nan for k in result if k in keys_to_check])
stim = VideoStim(join(VIDEO_DIR, 'obama_speech.mp4'))
result = ext.transform(stim).to_df()
keys_to_check = ['top_row', 'left_col', 'bottom_row', 'right_col']
assert [k in result.keys() for k in keys_to_check]
def test_google_video_api_shot_extractor(caplog):
ext = GoogleVideoAPIShotDetectionExtractor(request_rate=3)
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'))
result = ext.transform(stim).to_df()
log_message = caplog.records[-1].message
incomplete = (log_message == ("The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 90))
if not incomplete:
assert result.shape == (1, 5)
assert result['onset'][0] == 0.0
assert np.isclose(result['duration'][0], stim.duration, 0.1)
assert 'shot_id' in result.columns
assert result['shot_id'][0] == 0
ext = GoogleVideoAPIShotDetectionExtractor()
stim = VideoStim(join(VIDEO_DIR, 'shot_change.mp4'))
result = ext.transform(stim).to_df()
log_message = caplog.records[-1].message
incomplete = (log_message == ("The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 90))
if not incomplete:
assert result.shape == (2, 5)
assert np.isclose(result['onset'][1], 3.2, 0.1)
assert 'shot_id' in result.columns
assert result['shot_id'][1] == 1
def test_google_vision_face_batch():
obama_file = join(get_test_data_path(), 'image', 'obama.jpg')
people_file = join(get_test_data_path(), 'image', 'thai_people.jpg')
stims = [ImageStim(obama_file), ImageStim(people_file)]
ext = GoogleVisionAPIFaceExtractor(handle_annotations='first')
result = ext.transform(stims)
result = ExtractorResult.merge_stims(result)
assert 'face1_joyLikelihood' in result.columns
assert result['face1_joyLikelihood'][0] == 'VERY_LIKELY'
assert result['face1_joyLikelihood'][1] == 'VERY_LIKELY'
video = VideoStim(join(get_test_data_path(), 'video', 'obama_speech.mp4'))
conv = FrameSamplingFilter(every=10)
video = conv.transform(video)
result = ext.transform(video)
result = ExtractorResult.merge_stims(result)
assert 'face1_joyLikelihood' in result.columns
assert result.shape == (11, 137)
video = VideoStim(join(get_test_data_path(), 'video', 'small.mp4'))
video = conv.transform(video)
result = ext.transform(video)
result = ExtractorResult.merge_stims(result)
assert 'face1_joyLikelihood' not in result.columns
assert result.shape == (17, 7)
def test_google_vision_face_batch():
stims = ['apple', 'obama', 'thai_people']
stim_files = [join(get_test_data_path(), 'image', '%s.jpg' % s)
for s in stims]
stims = [ImageStim(s) for s in stim_files]
ext = GoogleVisionAPIFaceExtractor(batch_size=5)
result = ext.transform(stims)
result = merge_results(result, format='wide', extractor_names=False,
handle_annotations='first')
assert result.shape == (2, 139)
assert 'joyLikelihood' in result.columns
assert result['joyLikelihood'][0] == 'VERY_LIKELY'
assert result['joyLikelihood'][1] == 'VERY_LIKELY'
video = VideoStim(join(VIDEO_DIR, 'obama_speech.mp4'))
conv = FrameSamplingFilter(every=10)
video = conv.transform(video)
result = ext.transform(video)
result = merge_results(result, format='wide', extractor_names=False)
assert 'joyLikelihood' in result.columns
assert result.shape == (22, 139)
video = VideoStim(join(VIDEO_DIR, 'small.mp4'))
video = conv.transform(video)
result = ext.transform(video)
result = merge_results(result, format='wide', extractor_names=False)
assert 'joyLikelihood' not in result.columns
assert len(result) == 0
def test_frame_sampling_video_filter2():
filename = join(VIDEO_DIR, 'obama_speech.mp4')
video = VideoStim(filename, onset=4.2)
assert video.fps == 12
assert video.n_frames == 105
# Test frame indices
conv = FrameSamplingFilter(every=3)
derived = conv.transform(video)
assert derived.n_frames == 35
assert derived.frame_index[4] == 12
conv = FrameSamplingFilter(hertz=3)
derived = conv.transform(video)
assert derived.n_frames == 27
assert derived.frame_index[3] == 12
conv = FrameSamplingFilter(hertz=24)
derived = conv.transform(video)
assert derived.n_frames == 210
assert derived.frame_index[4] == 2
video.fps = 11.8
conv = FrameSamplingFilter(hertz=1)
derived = conv.transform(video)
assert derived.n_frames == 9
assert derived.frame_index[4] == 47
assert derived.frame_index[5] == 59
def test_frame_sampling_video_filter():
filename = join(VIDEO_DIR, 'small.mp4')
video = VideoStim(filename, onset=4.2)
assert video.fps == 30
assert video.n_frames in (167, 168)
assert video.width == 560
# Test frame filters
conv = FrameSamplingFilter(every=3)
derived = conv.transform(video)
assert derived.n_frames == math.ceil(video.n_frames / 3.0)
assert derived.duration == video.duration
first = next(f for f in derived)
assert type(first) == VideoFrameStim
assert first.name == 'frame[0]'
assert first.onset == 4.2
assert first.duration == 3 * (1 / 30.0)
second = [f for f in derived][1]
assert second.onset == 4.3
with pytest.raises(TypeError):
derived.get_frame(onset=1.0)
# Commented out because no longer allowing sampling filter chaining
# conv = FrameSamplingFilter(hertz=15)
# derived = conv.transform(derived)
# assert derived.n_frames == math.ceil(video.n_frames / 6.0)
# first = next(f for f in derived)
# assert type(first) == VideoFrameStim
# assert first.duration == 3 * (1 / 15.0)
# second = [f for f in derived][1]
# assert second.onset == 4.4
with pytest.raises(TypeError):
conv.transform(derived)
def resample_video(video_file, sampling_rate):
"""
This function resamples a video to the desired sampling rate. Can be useful for making video with high sampling
rates more tractable for analysis.
Parameters
----------
video_file: str
file path to video to be resampled.
sampling_rate: float
Desired sampling rate in Hz
Returns
-------
resampled_video: pliers video object with resampled video frames
"""
from pliers.stimuli import VideoStim
from pliers.filters import FrameSamplingFilter
video = VideoStim(video_file)
resamp_filter = FrameSamplingFilter(hertz=sampling_rate)
resampled_video = resamp_filter.transform(video)
return resampled_video
def test_optical_flow_extractor():
pytest.importorskip('cv2')
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'), onset=4.2)
result = FarnebackOpticalFlowExtractor().transform(stim).to_df()
target = result.query('onset==7.2')['total_flow']
# Value returned by cv2 seems to change over versions, so use low precision
assert np.isclose(target, 86248.05, 1e-4)
def test_compound_stim():
audio_dir = join(get_test_data_path(), 'audio')
audio = AudioStim(join(audio_dir, 'crowd.mp3'))
image1 = ImageStim(join(get_test_data_path(), 'image', 'apple.jpg'))
image2 = ImageStim(join(get_test_data_path(), 'image', 'obama.jpg'))
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename)
text = ComplexTextStim(text="The quick brown fox jumped...")
stim = CompoundStim([audio, image1, image2, video, text])
assert len(stim.elements) == 5
assert isinstance(stim.video, VideoStim)
assert isinstance(stim.complex_text, ComplexTextStim)
assert isinstance(stim.image, ImageStim)
with pytest.raises(AttributeError):
stim.nonexistent_type
assert stim.video_frame is None
# Test iteration
len([e for e in stim]) == 5
imgs = stim.get_stim(ImageStim, return_all=True)
assert len(imgs) == 2
assert all([isinstance(im, ImageStim) for im in imgs])
also_imgs = stim.get_stim('image', return_all=True)
assert imgs == also_imgs
def test_big_pipeline():
pytest.importorskip('pygraphviz')
filename = join(get_test_data_path(), 'video', 'obama_speech.mp4')
video = VideoStim(filename)
visual_nodes = [(FrameSamplingFilter(every=15), [
(TesseractConverter(), [LengthExtractor()]),
VibranceExtractor(),
'BrightnessExtractor',
])]
audio_nodes = [(VideoToAudioConverter(),
[WitTranscriptionConverter(),
'LengthExtractor'], 'video_to_audio')]
graph = Graph()
graph.add_nodes(visual_nodes)
graph.add_nodes(audio_nodes)
results = graph.run(video, merge=False)
result = merge_results(results, format='wide', extractor_names='multi')
# Test that pygraphviz outputs a file
drawfile = next(tempfile._get_candidate_names())
graph.draw(drawfile)
assert exists(drawfile)
os.remove(drawfile)
assert ('LengthExtractor', 'text_length') in result.columns
assert ('VibranceExtractor', 'vibrance') in result.columns
# assert not result[('onset', '')].isnull().any()
assert 'text[negotiations]' in result['stim_name'].values
assert 'frame[90]' in result['stim_name'].values
def test_google_video_api_extractor(caplog):
ext = GoogleVideoIntelligenceAPIExtractor(timeout=1)
stim = VideoStim(join(VIDEO_DIR, 'park.mp4'))
result = ext.transform(stim)
log_message = caplog.records[-1].message
assert log_message == ("The extraction reached the timeout limit of %fs, "
"which means the API may not have finished analyzing the "
"video and the results may be empty or incomplete." % 1.0)
ext = GoogleVideoIntelligenceAPIExtractor(timeout=500,
features=['LABEL_DETECTION',
'SHOT_CHANGE_DETECTION'])
result = ext.transform(stim).to_df()
log_message = caplog.records[-1].message
incomplete = (log_message == ("The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 500))
if not incomplete:
assert result.shape == (1, 31)
assert result['onset'][0] == 0.0
assert result['duration'][0] > 0.5 and result['duration'][0] < 0.6
assert result['category_plant'][0] > 0.5
assert result['park'][0] > 0.5
assert result['shot_id'][0] == 0
def test_video_frame_stim():
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename, onset=4.2)
frame = VideoFrameStim(video, 42)
assert frame.onset == (5.6)
assert np.array_equal(frame.data, video.get_frame(index=42).data)
assert frame.name == 'frame[42]'
def test_derived_video_converter():
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename, onset=4.2)
assert video.fps == 30
assert video.n_frames in (167, 168)
assert video.width == 560
# Test frame filters
conv = FrameSamplingConverter(every=3)
derived = conv.transform(video)
assert len(derived._frames) == math.ceil(video.n_frames / 3.0)
first = next(f for f in derived)
assert type(first) == VideoFrameStim
assert first.name == 'frame[0]'
assert first.onset == 4.2
assert first.duration == 3 * (1 / 30.0)
second = [f for f in derived][1]
assert second.onset == 4.3
# Should refilter from original frames
conv = FrameSamplingConverter(hertz=15)
derived = conv.transform(derived)
assert len(derived._frames) == math.ceil(video.n_frames / 6.0)
first = next(f for f in derived)
assert type(first) == VideoFrameStim
assert first.duration == 3 * (1 / 15.0)
second = [f for f in derived][1]
assert second.onset == 4.4
def test_multistep_converter():
conv = VideoToTextConverter()
filename = join(get_test_data_path(), 'video', 'obama_speech.mp4')
stim = VideoStim(filename)
text = conv.transform(stim)
assert isinstance(text, ComplexTextStim)
first_word = next(w for w in text)
assert type(first_word) == TextStim
def test_video_to_audio_converter():
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename)
conv = VideoToAudioConverter()
audio = conv.transform(video)
assert audio.history.source_class == 'VideoStim'
assert audio.history.source_file == filename
assert np.isclose(video.duration, audio.duration, 1e-2)
def test_optical_flow_extractor():
pytest.importorskip('cv2')
video_dir = join(get_test_data_path(), 'video')
stim = VideoStim(join(video_dir, 'small.mp4'))
result = DenseOpticalFlowExtractor().transform(stim).to_df()
target = result.query('onset==3.0')['total_flow']
# Value returned by cv2 seems to change over versions, so use low precision
assert np.isclose(target, 86248.05, 1e-4)
def test_derived_video_converter_cv2():
pytest.importorskip('cv2')
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename)
conv = FrameSamplingConverter(top_n=5)
derived = conv.transform(video)
assert len(derived._frames) == 5
assert type(next(f for f in derived)) == VideoFrameStim
def test_implicit_stim_conversion3():
video_dir = join(get_test_data_path(), 'video')
stim = VideoStim(join(video_dir, 'obama_speech.mp4'))
ext = LengthExtractor()
result = ext.transform(stim)
first_word = result[0].to_df()
# The word should be "today"
assert 'text_length' in first_word.columns
assert first_word['text_length'][0] == 5
def test_video_to_audio_converter():
filename = join(VIDEO_DIR, 'small.mp4')
video = VideoStim(filename, onset=4.2)
conv = VideoToAudioConverter()
audio = conv.transform(video)
assert audio.history.source_class == 'VideoStim'
assert audio.history.source_file == filename
assert audio.onset == 4.2
assert np.isclose(video.duration, audio.duration, 1e-2)
def test_frame_sampling_cv2():
pytest.importorskip('cv2')
filename = join(VIDEO_DIR, 'small.mp4')
video = VideoStim(filename)
conv = FrameSamplingFilter(top_n=5)
derived = conv.transform(video)
assert derived.n_frames == 5
assert type(next(f for f in derived)) == VideoFrameStim
def test_video_stim_bytestring():
path = join(get_test_data_path(), 'video', 'small.mp4')
vid = VideoStim(path)
assert vid._bytestring is None
bs = vid.get_bytestring()
assert isinstance(bs, str)
assert vid._bytestring is not None
raw = bs.encode()
with open(path, 'rb') as f:
assert raw == base64.b64encode(f.read())
def test_clarifai_api_video_extractor():
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'))
ext = ClarifaiAPIVideoExtractor()
assert ext.validate_keys()
result = ext.transform(stim).to_df()
assert result.shape == (6, 27)
assert result['toy'][0] > 0.5
assert result['onset'][1] == 1.0
assert result['duration'][0] == 1.0
assert np.isclose(result['duration'][5], 0.57)
def test_video_stim():
''' Test VideoStim functionality. '''
filename = join(get_test_data_path(), 'video', 'small.mp4')
video = VideoStim(filename, onset=4.2)
assert video.fps == 30
assert video.n_frames == 168
assert video.width == 560
assert video.duration == 5.57
# Test frame iterator
frames = [f for f in video]
assert len(frames) == 168
f1 = frames[100]
assert isinstance(f1, VideoFrameStim)
assert isinstance(f1.onset, float)
assert np.isclose(f1.duration, 1 / 30.0, 1e-5)
f1.data.shape == (320, 560, 3)
# Test getting of specific frame
f2 = video.get_frame(index=100)
assert isinstance(f2, VideoFrameStim)
assert isinstance(f2.onset, float)
assert f2.onset > 7.5
f2.data.shape == (320, 560, 3)
f2_copy = video.get_frame(onset=3.33334)
assert isinstance(f2, VideoFrameStim)
assert isinstance(f2.onset, float)
assert f2.onset > 7.5
assert np.array_equal(f2.data, f2_copy.data)
# Try another video
filename = join(get_test_data_path(), 'video', 'obama_speech.mp4')
video = VideoStim(filename)
assert video.fps == 12
assert video.n_frames == 105
assert video.width == 320
assert video.duration == 8.71
f3 = video.get_frame(index=104)
assert isinstance(f3, VideoFrameStim)
assert isinstance(f3.onset, float)
assert f3.duration > 0.0
assert f3.data.shape == (240, 320, 3)
def test_stim_history_tracking():
video = VideoStim(join(get_test_data_path(), 'video', 'obama_speech.mp4'))
assert video.history is None
conv = VideoToAudioConverter()
stim = conv.transform(video)
assert str(stim.history) == 'VideoStim->VideoToAudioConverter/AudioStim'
conv = WitTranscriptionConverter()
stim = conv.transform(stim)
assert str(
stim.history
) == 'VideoStim->VideoToAudioConverter/AudioStim->WitTranscriptionConverter/ComplexTextStim'
def test_google_video_api_explicit_extractor(caplog):
ext = GoogleVideoAPIExplicitDetectionExtractor(request_rate=3)
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'), onset=4.2)
result = ext.transform(stim).to_df()
log_message = caplog.records[-1].message
incomplete = (log_message == ("The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 90))
if not incomplete:
assert result.shape[1] == 5
assert result['onset'][0] >= 4.2
assert 'pornographyLikelihood' in result.columns
assert 'UNLIKELY' in result['pornographyLikelihood'][0]
def test_google_video_api_label_extractor(caplog):
ext = GoogleVideoAPILabelDetectionExtractor(mode='FRAME_MODE',
stationary_camera=True)
stim = VideoStim(join(VIDEO_DIR, 'small.mp4'))
ex_result = ext.transform(stim)
log_message = caplog.records[-1].message
incomplete = (log_message == (
"The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 90))
if not incomplete:
result = ex_result.to_df()
assert result.shape[1] > 20
assert 'category_toy' in result.columns
assert result['toy'][0] > 0.5
assert np.isclose(result['duration'][0], stim.duration, 0.1)
result = ex_result.to_df(format='long')
assert 'pornographyLikelihood' not in result['feature']
assert np.nan not in result['value']
ext = GoogleVideoAPILabelDetectionExtractor(mode='SHOT_MODE')
stim = VideoStim(join(VIDEO_DIR, 'shot_change.mp4'))
ex_result = ext.transform(stim)
log_message = caplog.records[-1].message
incomplete = (log_message == (
"The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 90))
if not incomplete:
raw = ex_result.raw['response']['annotationResults'][0]
assert 'shotLabelAnnotations' in raw
result = ex_result.to_df()
assert result.shape[1] > 10
assert result['onset'][1] == 0.0
assert np.isclose(result['onset'][2], 3.2, 0.1)
assert np.isnan(result['cat'][1])
assert result['cat'][2] > 0.5
assert np.isnan(result['clock'][2])
assert result['clock'][1] > 0.5 or result['clock'][0] > 0.5
def test_microsoft_vision_api_extractor_large():
default = config.get_option('allow_large_jobs')
default_large = config.get_option('large_job')
config.set_option('allow_large_jobs', False)
config.set_option('large_job', 3)
ext = MicrosoftVisionAPITagExtractor()
video = VideoStim(join(VIDEO_DIR, 'small.mp4'))
with pytest.raises(ValueError):
merge_results(ext.transform(video))
config.set_option('allow_large_jobs', default)
config.set_option('large_job', default_large)
def test_save():
text_dir = join(get_test_data_path(), 'text')
complextext_stim = ComplexTextStim(join(text_dir, 'complex_stim_no_header.txt'),
columns='ot', default_duration=0.2)
text_stim = TextStim(text='hello')
video_stim = VideoStim(join(get_test_data_path(), 'video', 'small.mp4'))
audio_stim = AudioStim(join(get_test_data_path(), 'audio', 'crowd.mp3'))
image_stim = ImageStim(join(get_test_data_path(), 'image', 'apple.jpg'))
stims = [complextext_stim, text_stim, video_stim, audio_stim, image_stim]
for s in stims:
path = tempfile.mktemp() + s._default_file_extension
s.save(path)
assert exists(path)
os.remove(path)
def test_video_trimming_filter():
video = VideoStim(join(VIDEO_DIR, 'small.mp4'))
filt = TemporalTrimmingFilter(end=4.0)
short_video = filt.transform(video)
assert short_video.fps == 30
assert short_video.duration == 4.0
frame_filt = VideoTrimmingFilter(end=100, frames=True)
short_video = frame_filt.transform(video)
assert short_video.fps == 30
assert short_video.n_frames == 100
error_filt = VideoTrimmingFilter(end=10.0, validation='strict')
with pytest.raises(ValueError):
short_video = error_filt.transform(video)
def test_google_video_api_extractor2(caplog):
segments = [{'startTimeOffset': '0.1s', 'endTimeOffset': '0.3s'},
{'startTimeOffset': '0.3s', 'endTimeOffset': '0.45s'}]
ext = GoogleVideoIntelligenceAPIExtractor(timeout=500, segments=segments,
features=['EXPLICIT_CONTENT_DETECTION'])
stim = VideoStim(join(VIDEO_DIR, 'park.mp4'))
result = ext.transform(stim).to_df()
log_message = caplog.records[-1].message
incomplete = (log_message == ("The extraction reached the timeout limit of"
" %fs, which means the API may not have finished analyzing the"
" video and the results may be empty or incomplete." % 500))
if not incomplete:
assert result.shape == (2, 5)
assert result['onset'][0] > 0.1 and result['onset'][0] < 0.3
assert result['onset'][1] > 0.3 and result['onset'][1] < 0.45
assert 'UNLIKELY' in result['pornographyLikelihood'][0]