def test_rms_extractor():
audio = AudioStim(join(AUDIO_DIR, 'barber.wav'))
ext = RMSExtractor()
df = ext.transform(audio).to_df()
assert df.shape == (1221, 5)
assert np.isclose(df['onset'][2], 0.092880)
assert np.isclose(df['duration'][2], 0.04644)
assert np.isclose(df['rms'][2], 0.229993)
assert np.isclose(df['onset'][4], 0.185760)
assert np.isclose(df['duration'][4], 0.04644)
assert np.isclose(df['rms'][4], 0.184349)
assert np.isclose(df['onset'][1219], 56.610249)
assert np.isclose(df['duration'][1219], 0.04644)
assert np.isclose(df['rms'][1219], 0.001348, rtol = 1e-03)
def test_resample():
ext = RMSExtractor()
res = ext.transform(join(get_test_data_path(), 'audio/homer.wav'))
df = res.to_df(format='long')
# Test downsample
downsampled_df = resample(df, 3)
assert np.allclose(downsampled_df.iloc[0].onset, 0)
assert np.allclose(downsampled_df.iloc[1].onset, 0.33333)
assert set(downsampled_df.columns) == {
'duration', 'onset', 'feature', 'value'}
assert downsampled_df['feature'].unique() == 'rms'
# This checks that the filtering has happened. If it has not, then
# this value for this frequency bin will be an alias and have a
# very different amplitude
assert downsampled_df[downsampled_df.onset == 0]['value'].values[0] != \
df[df.onset == 0]['value'].values[0]
assert downsampled_df[downsampled_df.onset == 0]['value'].values[0] != \
df[df.onset == 0]['value'].values[0]
assert np.allclose(downsampled_df[downsampled_df.onset == 2]['value'].values[0],
0.2261582761938699)
# Test upsample
ext = RMSExtractor(frame_length=1500, hop_length=1500,)
res = ext.transform(join(get_test_data_path(), 'audio/homer.wav'))
df = res.to_df(format='long')
upsampled_df = resample(df, 10)
assert np.allclose(upsampled_df.iloc[0].onset, 0)
assert np.allclose(upsampled_df.iloc[1].onset, 0.1)
assert set(upsampled_df.columns) == {
'duration', 'onset', 'feature', 'value'}
assert upsampled_df['feature'].unique() == 'rms'
# This checks that the filtering has happened. If it has not, then
# this value for this frequency bin will be an alias and have a
# very different amplitude
assert upsampled_df[upsampled_df.onset == 0]['value'].values[0] != \
df[df.onset == 0]['value'].values[0]
assert upsampled_df[upsampled_df.onset == 0]['value'].values[0] != \
df[df.onset == 0]['value'].values[0]
# Value will be slightly different at 2s with different sampling
assert np.allclose(
upsampled_df[upsampled_df.onset == 2]['value'].values[0], 0.25309)
def extract_audio_features(in_file):
"""
This function extracts audio intensity, tempo, and beats from the audio of a video file using the pliers library.
If you use this function, please cite the pliers library directly:
https://github.com/PsychoinformaticsLab/pliers#how-to-cite
Parameters
----------
in_file : str
file path to video or audio file to be processed
Returns
-------
low_level_audio_df : DataFrame
Pandas dataframe with a column per low-level feature (index is time).
"""
# compute tempo on a continuous basis
print('Extracting dynamic tempo...')
y, sr = librosa.load(in_file)
onset_env = librosa.onset.onset_strength(y, sr=sr)
time_seconds = np.arange(1, len(y)) / sr
dtempo = librosa.beat.tempo(onset_envelope=onset_env,
sr=sr,
ac_size=10,
aggregate=None)
n_samples = len(dtempo)
dtempo_samp_rate = time_seconds[-1] / n_samples
time_ms = np.arange(0, time_seconds[-1], dtempo_samp_rate) * 1000
dtempo_df = pd.DataFrame(dtempo,
columns=['dynamic_tempo'],
index=pd.to_datetime(time_ms, unit='ms'))
resamp_dtempo_df = dtempo_df.resample('10ms').mean()
if resamp_dtempo_df['dynamic_tempo'].isnull().values.any():
resamp_dtempo_df = dtempo_df.resample('10ms').bfill()
# audio RMS to capture changes in intensity
print('Extracting loudness...')
from pliers.extractors import RMSExtractor
rmsext = RMSExtractor()
rmsres = rmsext.transform(in_file)
rmsres_df = rmsres.to_df()
# Identify major beats in audio
print('Extracting major music beats...')
from pliers.extractors import BeatTrackExtractor
btext = BeatTrackExtractor()
bteres = btext.transform(in_file)
bteres_df = bteres.to_df()
# combine features into one dataframe
print('Aggregating data...')
low_level_audio_df = pd.DataFrame()
low_level_audio_df['onset_ms'] = rmsres_df['onset'] * 1000
low_level_audio_df['rms'] = rmsres_df['rms']
low_level_audio_df['beats'] = 0
for b in bteres_df['beat_track']:
low_level_audio_df.loc[b, 'beats'] = 1
low_level_audio_df.index = pd.to_datetime(rmsres_df['onset'], unit='s')
low_level_audio_df = low_level_audio_df.resample('10ms').mean()
low_level_audio_df['onset_ms'] = low_level_audio_df[
'onset_ms'] - low_level_audio_df['onset_ms'][0]
low_level_audio_df['dynamic_tempo'] = resamp_dtempo_df['dynamic_tempo']
low_level_audio_df.index.name = None
print('Auditory feature extraction complete.')
return low_level_audio_df
# Getting Started
The best way to see what *pliers* can offer is to jump right into an example.
## Example 1: Audio RMS
A measure that is likely to capture a large amount of variance in auditory cortex activity during naturalistic stimulation is the power of the signal in the auditory track. A simple way to measure this is by extracting the `Root-Mean-Square (RMS)` of the audio signal across time.
*Pliers* makes this very easy. All we need to do is to import the `RMSExtractor` object, and apply it to the `ParanoiaStory` audio stimulus.
from pliers.extractors import RMSExtractor
# Create an instance of this extractor
ext = RMSExtractor()
# Extract features from the audio stimulus
rms_result = ext.transform(paranoia_audio)
The Extractor returns an `ExtractorResult` object which contains the extracted values.
We can easily convert this to a *Pandas DataFrame*, a familar format that could easily be fed into a data-analysis pipeline, and is easy to inspect.
rms_df = rms_result.to_df()
rms_df
We can then easily plot the timeline of `rms` across time for the *ParanoiaStory* study.
rms_df.plot('onset', 'rms')