def load_image_luminance(image_files, hdim=None, vdim=None): '''Load a set of RGB images and return its luminance representation. Parameters ---------- image_files : list-like, (nimages,) A list of file names. The images should be in RGB uint8 format. vdim : int, optional hdim : int, optional Vertical and horizontal dimensions, respectively. If provided the images will be scaled to this size. Returns ------- arr : 3D np.array (nimages, vdim, hdim) The luminance representation of the images. ''' if (hdim and vdim): loader = lambda stim, sz: resize_image(stim, sz) else: loader = lambda stim, sz: np.asarray(stim) stimuli = [] for fdx, fl in iterator_func(enumerate(image_files), "load_image_luminance", total=len(image_files)): stimulus = Image.open(fl) stimulus = loader(stimulus, (vdim, hdim)) stimulus = rgb2lab(stimulus / 255.)[..., 0] stimuli.append(stimulus) return np.asarray(stimuli)
def video2luminance(video_file, size=None, nimages=np.inf): '''Convert the video frames to luminance images. Internally, this function loads one video frame into memory at a time. Tt converts the RGB pixel values from one frame to CIE-LAB pixel values. It then keeps the luminance channel only. This process is performed for all frames requested or until we reach the end of the video file. Parameters ---------- video_file : str Full path to the video file. This can be a video file on disk or from a website. size : optional, tuple (vdim, hdim) The desired output image size. If specified, the image is scaled or shrunk to this size. If not specified, the original size is kept. nimages : optional, int If specified, only `nimages` frames are converted to luminance. Returns ------- luminance_images : 3D np.ndarray, (nimages, vdim, hdim) Pixel values are in the 0-100 range. ''' vbuffer = video_buffer(video_file, nimages=nimages) luminance_video = [] for imageidx, image_rgb in iterator_func(enumerate(vbuffer), '%s.video2luminance' % __name__): luminance_image = imagearray2luminance(image_rgb, size=size).squeeze() luminance_video.append(luminance_image) return np.asarray(luminance_video)
def video2grey(video_file, size=None, nimages=np.inf): '''Convert the video frames to greyscale images. This function computes the mean across RGB color channels. Parameters ---------- video_file : str Full path to the video file. This can be a video file on disk or from a website. size : optional, tuple (vdim, hdim) The desired output image size. If specified, the image is scaled or shrunk to this size. If not specified, the original size is kept. nimages : optional, int If specified, only `nimages` frames are converted to greyscale. Returns ------- greyscale_images : 3D np.ndarray, (nimages, vdim, hdim) Pixel values are in the 0-1 range. ''' vbuffer = video_buffer(video_file, nimages=nimages) grey_video = [] for imageidx, image_rgb in iterator_func(enumerate(vbuffer), '%s.video2grey' % __name__): if size is not None: image_rgb = resize_image(image_rgb, size=size) grey_image = image_rgb.mean(-1) / 255.0 grey_video.append(grey_image) return np.asarray(grey_video)
def raw_project_stimulus(stimulus, filters, vhsize=(), dtype='float32'): '''Obtain responses to the stimuli from all filter quadrature-pairs. Parameters ---------- stimulus : np.ndarray, (nimages, vdim, hdim) or (nimages, npixels) The movie frames. If `stimulus` is two-dimensional with shape (nimages, npixels), then `vhsize=(vdim,hdim)` is required and `npixels == vdim*hdim`. Returns ------- output_sin : np.ndarray, (nimages, nfilters) output_cos : np.ndarray, (nimages, nfilters) ''' # parameters if stimulus.ndim == 3: nimages, vdim, hdim = stimulus.shape stimulus = stimulus.reshape(stimulus.shape[0], -1) vhsize = (vdim, hdim) # checks for 2D stimuli assert stimulus.ndim == 2 # (nimages, pixels) assert isinstance(vhsize, tuple) and len(vhsize) == 2 # (hdim, vdim) assert np.product(vhsize) == stimulus.shape[1] # hdim*vdim == pixels # Compute responses nfilters = len(filters) nimages = stimulus.shape[0] sin_responses = np.zeros((nimages, nfilters), dtype=dtype) cos_responses = np.zeros((nimages, nfilters), dtype=dtype) for gaborid, gabor_parameters in iterator_func(enumerate(filters), 'project_stimulus', total=len(filters)): sgabor0, sgabor90, tgabor0, tgabor90 = mk_3d_gabor(vhsize, **gabor_parameters) channel_sin, channel_cos = dotdelay_frames(sgabor0, sgabor90, tgabor0, tgabor90, stimulus) sin_responses[:, gaborid] = channel_sin cos_responses[:, gaborid] = channel_cos return sin_responses, cos_responses
def project_stimulus(stimulus, filters, quadrature_combination=sqrt_sum_squares, output_nonlinearity=log_compress, vhsize=(), dtype='float32'): '''Compute the motion energy filter responses to the stimuli. Parameters ---------- stimulus : np.ndarray, (nimages, vdim, hdim) or (nimages, npixels) The movie frames. If `stimulus` is two-dimensional with shape (nimages, npixels), then `vhsize=(vdim,hdim)` is required and `npixels == vdim*hdim`. Returns ------- filter_responses : np.ndarray, (nimages, nfilters) ''' # parameters if stimulus.ndim == 3: nimages, vdim, hdim = stimulus.shape stimulus = stimulus.reshape(stimulus.shape[0], -1) vhsize = (vdim, hdim) # checks for 2D stimuli assert stimulus.ndim == 2 # (nimages, pixels) assert isinstance(vhsize, tuple) and len(vhsize) == 2 # (hdim, vdim) assert np.product(vhsize) == stimulus.shape[1] # hdim*vdim == pixels # Compute responses nfilters = len(filters) nimages = stimulus.shape[0] filter_responses = np.zeros((nimages, nfilters), dtype=dtype) for gaborid, gabor_parameters in iterator_func(enumerate(filters), 'project_stimulus', total=len(filters)): sgabor0, sgabor90, tgabor0, tgabor90 = mk_3d_gabor(vhsize, **gabor_parameters) channel_sin, channel_cos = dotdelay_frames(sgabor0, sgabor90, tgabor0, tgabor90, stimulus) channel_response = quadrature_combination(channel_sin, channel_cos) channel_response = output_nonlinearity(channel_response) filter_responses[:, gaborid] = channel_response return filter_responses
def compute_temporal_pcs(self, generator=None, skip_first=False): '''Extract the temporal principal components of the total motion energy. Parameters ---------- generator : optional The video frame difference generator. Defaults to the ``video_file`` used to instantiate the class. skip_first : optional, bool By default, set the first timepoint of all the PCs is set to zeros because the first timepoint corresponds to the difference between the first frame and nothing. If `skip_first=True`, then the first frame is removed from the timecourse. Attributes ---------- decomposition_temporal_pcs : list, (ntimepoints, npcs) The temporal compoonents are scaled by their singular values (:math:`US`). ''' import moten.utils if generator is None: # allow the user to provide their own frame difference generator generator = self.get_frame_difference_generator() if skip_first: # drop the first frame b/c the difference is with 0's # and so projection is with itself generator.__next__() self.decomposition_temporal_pcs = [] ## TODO: batch for faster performance for frame_diff in iterator_func(generator, 'projecting stimuli', unit='[frames]'): # flatten and square frame_diff = self.output_nonlinearity(frame_diff.reshape(1, -1)) projection = frame_diff @ self.decomposition_spatial_pcs self.decomposition_temporal_pcs.append(projection.squeeze()) if skip_first is False: # if not skipped, set the first frame to 0s # b/c its difference is not really defined self.decomposition_temporal_pcs[0][:] *= 0
def compute_filter_responses(stimulus, stimulus_fps, aspect_ratio='auto', filter_temporal_width='auto', quadrature_combination=sqrt_sum_squares, output_nonlinearity=log_compress, dozscore=True, dtype=np.float64, pyramid_parameters={}): """Compute the motion energy filters' response to the stimuli. Parameters ---------- stimulus : 3D np.array (n, vdim, hdim) The movie frames. stimulus_fps : scalar The temporal frequency of the stimulus aspect_ratio : bool, or scalar Defaults to hdim/vdim. Otherwise, pass as scalar filter_temporal_width : int, None The number of frames in one filter. Defaults to approximately 0.666[secs] (floor(stimulus_fps*(2/3))). quadrature_combination : function, optional Specifies how to combine the channel reponses quadratures. The function must take the sin and cos as arguments in order. Defaults to: (sin^2 + cos^2)^1/2 output_nonlinearity : function, optional Passes the channels (after `quadrature_combination`) through a non-linearity. The function input is the (`n`,`nfilters`) array. Defaults to: ln(x + 1e-05) dozscore : bool, optional Whether to z-score the channel responses in time dtype : np.dtype Defaults to np.float64 pyramid_parameters: dict See :func:`mk_moten_pyramid_params` for details on parameters specifiying a motion energy pyramid. Returns ------- filter_responses : np.array, (n, nfilters) """ nimages, vdim, hdim = stimulus.shape stimulus = stimulus.reshape(stimulus.shape[0], -1) vhsize = (vdim, hdim) if aspect_ratio == 'auto': aspect_ratio = hdim/float(vdim) if filter_temporal_width == 'auto': filter_temporal_width = int(stimulus_fps*(2./3.)) # pass parameters pkwargs = dict(aspect_ratio=aspect_ratio, filter_temporal_width=filter_temporal_width) pkwargs.update(**pyramid_parameters) parameter_names, gabor_parameters = mk_moten_pyramid_params(stimulus_fps, **pkwargs) ngabors = gabor_parameters.shape[0] filters = [{name : gabor_parameters[idx, pdx] for pdx, name \ in enumerate(parameter_names)} \ for idx in range(ngabors)] info = 'Computing responses for #%i filters across #%i images (aspect_ratio=%0.03f)' print(info%(len(gabor_parameters), nimages, aspect_ratio)) channels = np.zeros((nimages, len(gabor_parameters)), dtype=dtype) for idx, gabor_param_dict in iterator_func(enumerate(filters), '%s.compute_filter_responses'%__name__, total=len(filters)): gabor = mk_3d_gabor(vhsize, **gabor_param_dict) gabor0, gabor90, tgabor0, tgabor90 = gabor channel_sin, channel_cos = dotdelay_frames(gabor0, gabor90, tgabor0, tgabor90, stimulus, ) channel = quadrature_combination(channel_sin, channel_cos) channels[:,idx] = channel channels = output_nonlinearity(channels) if dozscore: from scipy.stats import zscore channels = zscore(channels) return channels
def compute_spatial_gabor_responses(stimulus, aspect_ratio='auto', spatial_frequencies=[0,2,4,8,16,32], quadrature_combination=sqrt_sum_squares, output_nonlinearity=log_compress, dtype=np.float64, dozscore=True): """Compute the spatial gabor filters' response to each stimulus. Parameters ---------- stimulus : 3D np.array (n, vdim, hdim) The stimulus frames. spatial_frequencies : array-like The spatial frequencies to compute. The spatial envelope is determined by this. quadrature_combination : function, optional Specifies how to combine the channel reponses quadratures. The function must take the sin and cos as arguments in order. Defaults to: (sin^2 + cos^2)^1/2 output_nonlinearity : function, optional Passes the channels (after `quadrature_combination`) through a non-linearity. The function input is the (`n`,`nfilters`) array. Defaults to: ln(x + 1e-05) dozscore : bool, optional Whether to z-score the channel responses in time dtype : np.dtype Defaults to np.float64 Returns ------- filter_responses : np.array, (n, nfilters) """ nimages, vdim, hdim = stimulus.shape vhsize = (vdim, hdim) if aspect_ratio == 'auto': aspect_ratio = hdim/float(vdim) stimulus = stimulus.reshape(stimulus.shape[0], -1) parameter_names, gabor_parameters = mk_moten_pyramid_params( 1., # fps filter_temporal_width=1., aspect_ratio=aspect_ratio, temporal_frequencies=[0.], spatial_directions=[0.], spatial_frequencies=spatial_frequencies, ) ngabors = gabor_parameters.shape[0] filters = [{name : gabor_parameters[idx, pdx] for pdx, name \ in enumerate(parameter_names)} \ for idx in range(ngabors)] info = 'Computing responses for #%i filters across #%i images (aspect_ratio=%0.03f)' print(info%(len(gabor_parameters), nimages, aspect_ratio)) channels = np.zeros((nimages, len(gabor_parameters)), dtype=dtype) for idx, gabor_param_dict in iterator_func(enumerate(filters), '%s.compute_spatial_gabor_responses'%__name__, total=len(gabor_parameters)): sgabor_sin, sgabor_cos, _, _ = mk_3d_gabor(vhsize, **gabor_param_dict) channel_sin, channel_cos = dotspatial_frames(sgabor_sin, sgabor_cos, stimulus) channel = quadrature_combination(channel_sin, channel_cos) channels[:, idx] = channel channels = output_nonlinearity(channels) if dozscore: from scipy.stats import zscore channels = zscore(channels) return channels