def load_image_luminance(image_files, hdim=None, vdim=None):
'''Load a set of RGB images and return its luminance representation.
Parameters
----------
image_files : list-like, (nimages,)
A list of file names.
The images should be in RGB uint8 format.
vdim : int, optional
hdim : int, optional
Vertical and horizontal dimensions, respectively.
If provided the images will be scaled to this size.
Returns
-------
arr : 3D np.array (nimages, vdim, hdim)
The luminance representation of the images.
'''
if (hdim and vdim):
loader = lambda stim, sz: resize_image(stim, sz)
else:
loader = lambda stim, sz: np.asarray(stim)
stimuli = []
for fdx, fl in iterator_func(enumerate(image_files),
"load_image_luminance",
total=len(image_files)):
stimulus = Image.open(fl)
stimulus = loader(stimulus, (vdim, hdim))
stimulus = rgb2lab(stimulus / 255.)[..., 0]
stimuli.append(stimulus)
return np.asarray(stimuli)
def video2luminance(video_file, size=None, nimages=np.inf):
'''Convert the video frames to luminance images.
Internally, this function loads one video frame into memory at a time.
Tt converts the RGB pixel values from one frame to CIE-LAB pixel values.
It then keeps the luminance channel only. This process is performed
for all frames requested or until we reach the end of the video file.
Parameters
----------
video_file : str
Full path to the video file.
This can be a video file on disk or from a website.
size : optional, tuple (vdim, hdim)
The desired output image size.
If specified, the image is scaled or shrunk to this size.
If not specified, the original size is kept.
nimages : optional, int
If specified, only `nimages` frames are converted to luminance.
Returns
-------
luminance_images : 3D np.ndarray, (nimages, vdim, hdim)
Pixel values are in the 0-100 range.
'''
vbuffer = video_buffer(video_file, nimages=nimages)
luminance_video = []
for imageidx, image_rgb in iterator_func(enumerate(vbuffer),
'%s.video2luminance' % __name__):
luminance_image = imagearray2luminance(image_rgb, size=size).squeeze()
luminance_video.append(luminance_image)
return np.asarray(luminance_video)
def video2grey(video_file, size=None, nimages=np.inf):
'''Convert the video frames to greyscale images.
This function computes the mean across RGB color channels.
Parameters
----------
video_file : str
Full path to the video file.
This can be a video file on disk or from a website.
size : optional, tuple (vdim, hdim)
The desired output image size.
If specified, the image is scaled or shrunk to this size.
If not specified, the original size is kept.
nimages : optional, int
If specified, only `nimages` frames are converted to greyscale.
Returns
-------
greyscale_images : 3D np.ndarray, (nimages, vdim, hdim)
Pixel values are in the 0-1 range.
'''
vbuffer = video_buffer(video_file, nimages=nimages)
grey_video = []
for imageidx, image_rgb in iterator_func(enumerate(vbuffer),
'%s.video2grey' % __name__):
if size is not None:
image_rgb = resize_image(image_rgb, size=size)
grey_image = image_rgb.mean(-1) / 255.0
grey_video.append(grey_image)
return np.asarray(grey_video)
def raw_project_stimulus(stimulus,
filters,
vhsize=(),
dtype='float32'):
'''Obtain responses to the stimuli from all filter quadrature-pairs.
Parameters
----------
stimulus : np.ndarray, (nimages, vdim, hdim) or (nimages, npixels)
The movie frames.
If `stimulus` is two-dimensional with shape (nimages, npixels), then
`vhsize=(vdim,hdim)` is required and `npixels == vdim*hdim`.
Returns
-------
output_sin : np.ndarray, (nimages, nfilters)
output_cos : np.ndarray, (nimages, nfilters)
'''
# parameters
if stimulus.ndim == 3:
nimages, vdim, hdim = stimulus.shape
stimulus = stimulus.reshape(stimulus.shape[0], -1)
vhsize = (vdim, hdim)
# checks for 2D stimuli
assert stimulus.ndim == 2 # (nimages, pixels)
assert isinstance(vhsize, tuple) and len(vhsize) == 2 # (hdim, vdim)
assert np.product(vhsize) == stimulus.shape[1] # hdim*vdim == pixels
# Compute responses
nfilters = len(filters)
nimages = stimulus.shape[0]
sin_responses = np.zeros((nimages, nfilters), dtype=dtype)
cos_responses = np.zeros((nimages, nfilters), dtype=dtype)
for gaborid, gabor_parameters in iterator_func(enumerate(filters),
'project_stimulus',
total=len(filters)):
sgabor0, sgabor90, tgabor0, tgabor90 = mk_3d_gabor(vhsize, **gabor_parameters)
channel_sin, channel_cos = dotdelay_frames(sgabor0, sgabor90,
tgabor0, tgabor90,
stimulus)
sin_responses[:, gaborid] = channel_sin
cos_responses[:, gaborid] = channel_cos
return sin_responses, cos_responses
def project_stimulus(stimulus,
filters,
quadrature_combination=sqrt_sum_squares,
output_nonlinearity=log_compress,
vhsize=(),
dtype='float32'):
'''Compute the motion energy filter responses to the stimuli.
Parameters
----------
stimulus : np.ndarray, (nimages, vdim, hdim) or (nimages, npixels)
The movie frames.
If `stimulus` is two-dimensional with shape (nimages, npixels), then
`vhsize=(vdim,hdim)` is required and `npixels == vdim*hdim`.
Returns
-------
filter_responses : np.ndarray, (nimages, nfilters)
'''
# parameters
if stimulus.ndim == 3:
nimages, vdim, hdim = stimulus.shape
stimulus = stimulus.reshape(stimulus.shape[0], -1)
vhsize = (vdim, hdim)
# checks for 2D stimuli
assert stimulus.ndim == 2 # (nimages, pixels)
assert isinstance(vhsize, tuple) and len(vhsize) == 2 # (hdim, vdim)
assert np.product(vhsize) == stimulus.shape[1] # hdim*vdim == pixels
# Compute responses
nfilters = len(filters)
nimages = stimulus.shape[0]
filter_responses = np.zeros((nimages, nfilters), dtype=dtype)
for gaborid, gabor_parameters in iterator_func(enumerate(filters),
'project_stimulus',
total=len(filters)):
sgabor0, sgabor90, tgabor0, tgabor90 = mk_3d_gabor(vhsize, **gabor_parameters)
channel_sin, channel_cos = dotdelay_frames(sgabor0, sgabor90,
tgabor0, tgabor90,
stimulus)
channel_response = quadrature_combination(channel_sin, channel_cos)
channel_response = output_nonlinearity(channel_response)
filter_responses[:, gaborid] = channel_response
return filter_responses
def compute_temporal_pcs(self, generator=None, skip_first=False):
'''Extract the temporal principal components of the total motion energy.
Parameters
----------
generator : optional
The video frame difference generator. Defaults to the ``video_file``
used to instantiate the class.
skip_first : optional, bool
By default, set the first timepoint of all the PCs is set to zeros because
the first timepoint corresponds to the difference between the first frame and nothing.
If `skip_first=True`, then the first frame is removed from the timecourse.
Attributes
----------
decomposition_temporal_pcs : list, (ntimepoints, npcs)
The temporal compoonents are scaled by their singular values (:math:`US`).
'''
import moten.utils
if generator is None:
# allow the user to provide their own frame difference generator
generator = self.get_frame_difference_generator()
if skip_first:
# drop the first frame b/c the difference is with 0's
# and so projection is with itself
generator.__next__()
self.decomposition_temporal_pcs = []
## TODO: batch for faster performance
for frame_diff in iterator_func(generator,
'projecting stimuli',
unit='[frames]'):
# flatten and square
frame_diff = self.output_nonlinearity(frame_diff.reshape(1, -1))
projection = frame_diff @ self.decomposition_spatial_pcs
self.decomposition_temporal_pcs.append(projection.squeeze())
if skip_first is False:
# if not skipped, set the first frame to 0s
# b/c its difference is not really defined
self.decomposition_temporal_pcs[0][:] *= 0
def compute_filter_responses(stimulus,
stimulus_fps,
aspect_ratio='auto',
filter_temporal_width='auto',
quadrature_combination=sqrt_sum_squares,
output_nonlinearity=log_compress,
dozscore=True,
dtype=np.float64,
pyramid_parameters={}):
"""Compute the motion energy filters' response to the stimuli.
Parameters
----------
stimulus : 3D np.array (n, vdim, hdim)
The movie frames.
stimulus_fps : scalar
The temporal frequency of the stimulus
aspect_ratio : bool, or scalar
Defaults to hdim/vdim. Otherwise, pass as scalar
filter_temporal_width : int, None
The number of frames in one filter.
Defaults to approximately 0.666[secs] (floor(stimulus_fps*(2/3))).
quadrature_combination : function, optional
Specifies how to combine the channel reponses quadratures.
The function must take the sin and cos as arguments in order.
Defaults to: (sin^2 + cos^2)^1/2
output_nonlinearity : function, optional
Passes the channels (after `quadrature_combination`) through a
non-linearity. The function input is the (`n`,`nfilters`) array.
Defaults to: ln(x + 1e-05)
dozscore : bool, optional
Whether to z-score the channel responses in time
dtype : np.dtype
Defaults to np.float64
pyramid_parameters: dict
See :func:`mk_moten_pyramid_params` for details on parameters
specifiying a motion energy pyramid.
Returns
-------
filter_responses : np.array, (n, nfilters)
"""
nimages, vdim, hdim = stimulus.shape
stimulus = stimulus.reshape(stimulus.shape[0], -1)
vhsize = (vdim, hdim)
if aspect_ratio == 'auto':
aspect_ratio = hdim/float(vdim)
if filter_temporal_width == 'auto':
filter_temporal_width = int(stimulus_fps*(2./3.))
# pass parameters
pkwargs = dict(aspect_ratio=aspect_ratio,
filter_temporal_width=filter_temporal_width)
pkwargs.update(**pyramid_parameters)
parameter_names, gabor_parameters = mk_moten_pyramid_params(stimulus_fps,
**pkwargs)
ngabors = gabor_parameters.shape[0]
filters = [{name : gabor_parameters[idx, pdx] for pdx, name \
in enumerate(parameter_names)} \
for idx in range(ngabors)]
info = 'Computing responses for #%i filters across #%i images (aspect_ratio=%0.03f)'
print(info%(len(gabor_parameters), nimages, aspect_ratio))
channels = np.zeros((nimages, len(gabor_parameters)), dtype=dtype)
for idx, gabor_param_dict in iterator_func(enumerate(filters),
'%s.compute_filter_responses'%__name__,
total=len(filters)):
gabor = mk_3d_gabor(vhsize,
**gabor_param_dict)
gabor0, gabor90, tgabor0, tgabor90 = gabor
channel_sin, channel_cos = dotdelay_frames(gabor0, gabor90,
tgabor0, tgabor90,
stimulus,
)
channel = quadrature_combination(channel_sin, channel_cos)
channels[:,idx] = channel
channels = output_nonlinearity(channels)
if dozscore:
from scipy.stats import zscore
channels = zscore(channels)
return channels
def compute_spatial_gabor_responses(stimulus,
aspect_ratio='auto',
spatial_frequencies=[0,2,4,8,16,32],
quadrature_combination=sqrt_sum_squares,
output_nonlinearity=log_compress,
dtype=np.float64,
dozscore=True):
"""Compute the spatial gabor filters' response to each stimulus.
Parameters
----------
stimulus : 3D np.array (n, vdim, hdim)
The stimulus frames.
spatial_frequencies : array-like
The spatial frequencies to compute. The spatial envelope is determined by this.
quadrature_combination : function, optional
Specifies how to combine the channel reponses quadratures.
The function must take the sin and cos as arguments in order.
Defaults to: (sin^2 + cos^2)^1/2
output_nonlinearity : function, optional
Passes the channels (after `quadrature_combination`) through a
non-linearity. The function input is the (`n`,`nfilters`) array.
Defaults to: ln(x + 1e-05)
dozscore : bool, optional
Whether to z-score the channel responses in time
dtype : np.dtype
Defaults to np.float64
Returns
-------
filter_responses : np.array, (n, nfilters)
"""
nimages, vdim, hdim = stimulus.shape
vhsize = (vdim, hdim)
if aspect_ratio == 'auto':
aspect_ratio = hdim/float(vdim)
stimulus = stimulus.reshape(stimulus.shape[0], -1)
parameter_names, gabor_parameters = mk_moten_pyramid_params(
1., # fps
filter_temporal_width=1.,
aspect_ratio=aspect_ratio,
temporal_frequencies=[0.],
spatial_directions=[0.],
spatial_frequencies=spatial_frequencies,
)
ngabors = gabor_parameters.shape[0]
filters = [{name : gabor_parameters[idx, pdx] for pdx, name \
in enumerate(parameter_names)} \
for idx in range(ngabors)]
info = 'Computing responses for #%i filters across #%i images (aspect_ratio=%0.03f)'
print(info%(len(gabor_parameters), nimages, aspect_ratio))
channels = np.zeros((nimages, len(gabor_parameters)), dtype=dtype)
for idx, gabor_param_dict in iterator_func(enumerate(filters),
'%s.compute_spatial_gabor_responses'%__name__,
total=len(gabor_parameters)):
sgabor_sin, sgabor_cos, _, _ = mk_3d_gabor(vhsize,
**gabor_param_dict)
channel_sin, channel_cos = dotspatial_frames(sgabor_sin, sgabor_cos, stimulus)
channel = quadrature_combination(channel_sin, channel_cos)
channels[:, idx] = channel
channels = output_nonlinearity(channels)
if dozscore:
from scipy.stats import zscore
channels = zscore(channels)
return channels
