def gen_ret_vars(retfile,stimfile): needed_ret_var_names = ['pval_ret','ret'] needed_ret_var_vals = ut.loadmat(retfile,needed_ret_var_names) ret_vars = {key:val for key,val in zip(needed_ret_var_names,needed_ret_var_vals)} result = ut.loadmat(stimfile,'result')[()] ret_vars['position'] = result['position'] paramdict_normal = ut.loadmat(retfile,'paramdict_normal') ret_vars['paramdict_normal'] = ut.matfile_to_dict(paramdict_normal) return ret_vars
def gen_frm_on(frame_file, rg=(1, 0)): info = ut.loadmat(frame_file, 'info')[()] frame = info['frame'] frm = np.floor(frame[rg[0]:frame.size + rg[1]] / 4).astype('int') frm_on = np.concatenate( [np.arange(x, y) for x, y in zip(frm[0::2], frm[1::2])]) return frm_on
def get_nbydepth(datafiles): nbydepth = np.zeros((len(datafiles),)) for i,datafile in enumerate(datafiles): corrected = ut.loadmat(datafile,'corrected')[()] nbydepth[i] = corrected.shape[0] #with h5py.File(datafile,mode='r') as f: #nbydepth[i] = (f['corrected'][:].T.shape[0]) return nbydepth
def fix_up_directions(locy, locx, stimfile, datafile): # positive direction is temporal # positive direction is UP!! result = ut.matfile_to_dict(ut.loadmat(stimfile, 'result')) # [()] if 'range' in list(result.keys()): gridsize = 5 ctr = np.array( (result['range'][0:2].mean(), result['range'][2:].mean()) ) # ctr: x center of range, y center of range # fixed 18/10/30; for expts. before 18/10/30, this will have to be switched! else: gridsize = 10 ctr = np.array((0, 0)) Ny = locy.max() + 1 Nx = locx.max() + 1 xrg = np.arange(-(Nx - 1) * gridsize / 2, (Nx + 1) * gridsize / 2, gridsize) # positive direction is temporal yrg = -np.arange(-(Ny - 1) * gridsize / 2, (Ny + 1) * gridsize / 2, gridsize) # positive direction is UP!! # flipping for expts. before 18/10/30 toflip = int(datafile.split('/')[-4]) < 181030 if toflip: ctr = ctr * np.array((1, -1)) # inverting for expts. between 18/10/30 and 18/12/09 notquitefixed = int(datafile.split('/')[-4]) < 181209 if ~toflip and notquitefixed: yrg = -yrg xrg = xrg + ctr[0] yrg = yrg + ctr[1] locydeg = yrg[locy.astype('int')] locxdeg = xrg[locx.astype('int')] return locydeg, locxdeg
Wstar_dict = np.load(weights_files[iwt], allow_pickle=True)[()] losses[iwt] = Wstar_dict['loss'] weights_files = [ wf for wf, l in zip(weights_files, losses) if l < np.nanpercentile(losses, lcutoff) ] target_file = calnet_base + 'dynamics/vip_halo_l4_opto_tavg_connection_deletion_%s.npy' % fit_lbl build_models_and_simulate_opto_effects(weights_files, target_file, sim_options=sim_options, pool_size=pool_size) if __name__ == '__main__': rrs_orig_layer, rfs_orig_layer = ut.loadmat(vip_halo_l4_matfile, ['rrs', 'rfs']) scall_halo_l4 = opto_utils.norm_to_mean_light_off(rrs_orig_layer[0, 0]) opto_slope, opto_intercept = compute_opto_line(scall_halo_l4) fit_lbl = sys.argv[1] if len(sys.argv) == 3: pool_size = int(sys.argv[2]) run(fit_lbl, pool_size=pool_size) elif len(sys.argv) > 3: pool_size = int(sys.argv[2]) calnet_base = sys.argv[3] run(fit_lbl, pool_size=pool_size, calnet_base=calnet_base) else: run(fit_lbl)
def analyze_precise_retinotopy(datafiles, stimfile, retfile, criterion=lambda x: x > 100, rg=(2, -10), nbefore=nbefore, nafter=nafter, gridsize=10): nbydepth = np.zeros((len(datafiles), )) for i, datafile in enumerate(datafiles): corrected = ut.loadmat(datafile, 'corrected') nbydepth[i] = corrected.shape[0] # with h5py.File(datafile,mode='r') as f: # nbydepth[i] = (f['corrected'][:].T.shape[0]) trialwise, ctrialwise, strialwise, dfof, straces, dtrialwise, trialwise_t_offset = ut.gen_precise_trialwise( datafiles, rg=rg, nbefore=nbefore, nafter=nafter) zstrialwise = sst.zscore(strialwise.reshape( (strialwise.shape[0], -1)).T).T.reshape(strialwise.shape) result = sio.loadmat(stimfile, squeeze_me=True)['result'][()] infofile = sio.loadmat(datafiles[0][:-12] + '.mat', squeeze_me=True) #retfile = sio.loadmat(retfile,squeeze_me=True) locinds = result['locinds'] #retfile['locinds'] has_inverse = False try: # inverted = result['inverted'][()] inverted = np.tile(result['inverted'], (result['repetitions'], )) has_inverse = True except: has_inverse = False frame = infofile['info'][()]['frame'][()] frame = np.unique( frame[rg[0]:frame.size + rg[1]] ) # this format for all the retinotopic mapping through 12/12 data = strialwise[:, :, nbefore:strialwise.shape[-1] - nafter] Ny = locinds[:, 0].max() Nx = locinds[:, 1].max() try: try: dxdt = sio.loadmat(datafiles[1], squeeze_me=True)['dxdt'] except: with h5py.File(datafiles[1], mode='r') as f: dxdt = f['dxdt'][:].T except: print('no running data saved; assuming all running') dxdt = 101 * np.ones((frame.max(), )) trialrun = np.zeros(frame[0::2].shape) for i in range(len(trialrun)): trialrun[i] = dxdt[frame[0::2][i]:frame[1::2][i]].mean() runtrial = criterion(np.abs(trialrun)) if has_inverse: ret = np.zeros((data.shape[0], Ny, Nx, 2)) for j in range(Ny): for k in range(Nx): lkat = np.logical_and( np.logical_and( np.logical_and(locinds[:, 0] == j + 1, locinds[:, 1] == k + 1), runtrial), np.nanmax(np.nanmax(data, 0), -1)) lkat_reg = np.logical_and(lkat, np.logical_not(inverted)) lkat_inv = np.logical_and(lkat, inverted) n_reg = lkat_reg.sum() n_inv = lkat_inv.sum() print((n_reg, n_inv)) for idx in np.where(lkat_reg)[0]: ret[:, j, k, 0] = ret[:, j, k, 0] + data[:, idx].mean(1) / n_reg for idx in np.where(lkat_inv)[0]: ret[:, j, k, 1] = ret[:, j, k, 1] + data[:, idx].mean(1) / n_inv assert (~np.isnan(np.nanmax(ret[:, j, k]))) else: ret = np.zeros((data.shape[0], Ny, Nx)) for j in range(Ny): for k in range(Nx): lkat_reg = np.logical_and( np.logical_and(locinds[:, 0] == j + 1, locinds[:, 1] == k + 1), runtrial) n_reg = lkat_reg.sum() print((n_reg, )) #n_inv)) for idx in np.where(lkat_reg)[0]: ret[:, j, k] = ret[:, j, k] + data[:, idx].mean(1) / n_reg assert (~np.isnan(np.nanmax(ret[:, j, k]))) if 'range' in result.dtype.names: gridsize = 5 ctr = np.array( (result['range'][0:2].mean(), -result['range'][2:].mean()) ) # ctr: x center of range, y center of range # fixed 18/10/30; for expts. after 18/10/30, this will have to be switched! else: gridsize = 10 ctr = np.array((0, 0)) # flipping for expts. after 18/10/30 toflip = int(datafiles[0].split('/')[-4]) > 181030 if toflip: ctr = ctr * np.array((1, -1)) xrg = np.arange(-(Nx - 1) * gridsize / 2, (Nx + 1) * gridsize / 2, gridsize) yrg = np.arange(-(Ny - 1) * gridsize / 2, (Ny + 1) * gridsize / 2, gridsize) # inverting for expts. before 18/12/09 notquitefixed = int(datafiles[0].split('/')[-4]) < 181209 if toflip and notquitefixed: yrg = -yrg if has_inverse: paramdict = [ ut.fit_2d_gaussian((xrg, yrg), ret[:, :, :, 0]), ut.fit_2d_gaussian((xrg, yrg), ret[:, :, :, 1]) ] paramdict[0]['gridsize'] = gridsize paramdict[1]['gridsize'] = gridsize paramdict[0]['ctr'] = ctr paramdict[1]['ctr'] = ctr else: paramdict = ut.fit_2d_gaussian( (xrg, yrg), ret) #,ut.fit_2d_gaussian((xrg,yrg),ret[:,:,:,1])] paramdict['gridsize'] = gridsize paramdict['ctr'] = ctr pval_ret = np.zeros(strialwise.shape[0]) for i in range(strialwise.shape[0]): _, pval_ret[i] = sst.ttest_rel(strialwise[i, :, nbefore - 1], strialwise[i, :, nbefore + 1]) return ret, paramdict, pval_ret, trialrun, has_inverse, locinds
(16, 1, 4, 8, 2))[np.newaxis]).sum(1), kind='stable') order2 = np.argsort(utils.nubs_active[order1][::-1].sum(1), kind='stable')[::-1] # order of stimuli for tuning curve display purposes evan_order_actual = order1[order2] # order of stimuli for stimulus identity display purposes evan_order_apparent = np.argsort(utils.nubs_active[::-1].sum(1), kind='stable')[::-1] nub_no = utils.nubs_active[evan_order_actual].sum(1) #parula = ListedColormap(ut.loadmat('/Users/dan/Documents/code/adesnal/matlab_parula_colormap.mat','cmap')) # similar to parula colormap, ported to python parula_path = '/Users/dan/Documents/code/adesnal/' parula_filename = parula_path + 'matlab_parula_colormap.mat' parula = ListedColormap(ut.loadmat(parula_filename, 'cmap')) def compute_tuning(dsname, keylist, datafield, run_fn, dilation_fn=dilation_fn, gen_nub_selector=utils.gen_nub_selector_v1, trialwise_dfof=False): # extract pandas dataframe from hdf5 file df, roi_info, trial_info = ut.compute_tavg_dataframe( dsname, 'nub_0', datafield=datafield, keylist=keylist,
def analyze(datafiles,stimfile,frame_adjust=None,rg=(1,0),nbefore=nbefore,nafter=nafter,stim_params=None): # stim_params: list (or similar) of tuples, where first element is a string corresponding to a field of the # output hdf5 file proc, and second element is a function taking result as an input, to yield the correct data # find number of ROIs in each plane nbydepth = get_nbydepth(datafiles) nplanes = len(nbydepth) # get trialized fluorescence data trialwise,ctrialwise,strialwise,dfof,straces,dtrialwise,proc1 = ut.gen_precise_trialwise(datafiles,rg=rg,frame_adjust=frame_adjust,nbefore=nbefore,nafter=nafter,blcutoff=blcutoff) # , trialwise_t_offset # load stimulus data #result = sio.loadmat(stimfile,squeeze_me=True)['result'][()] result = ut.loadmat(stimfile,'result')[()] #result = result[()] # correct stim trigger frames if necessary #infofile = sio.loadmat(datafiles[0][:-12]+'.mat',squeeze_me=True) # original .mat file info = ut.loadmat(datafiles[0][:-12]+'.mat','info')[()] # original .mat file #frame = infofile['info'][()]['frame'][()] frame = info['frame'][()].astype(np.int64) if not rg is None: frame = frame[rg[0]:frame.size+rg[1]] else: event_id = info['event_id'][()].astype(np.int64) frame = frame[event_id==1] if frame_adjust: frame = frame_adjust(frame) while np.min(np.diff(frame)) < 0: brk = np.argmin(np.diff(frame))+1 frame[brk:] = frame[brk:] + 65536 # load running and pupil data dxdt = ut.loadmat(datafiles[0],'dxdt').flatten() try: # first entry of pupil_ctr is x, second entry is y pupil_ctr,pupil_area,pupil_frac_ctr,pupil_frac_area = ut.loadmat(datafiles[0],['pupil_ctr','pupil_area','pupil_frac_ctr','pupil_frac_area']) pupil_area = pupil_area.flatten() pupil_frac_area = pupil_frac_area.flatten() except: print('no eye tracking data for ' + stimfile) pupil_ctr = None pupil_frac_ctr = None pupil_area = None pupil_frac_area = None nplanes = len(datafiles) msk,ctr = load_msk_ctr(datafiles[0]) cell_mask = np.zeros((0,)+msk.shape[1:],dtype='bool') cell_center = np.zeros((0,2)) cell_depth = np.zeros((0,)) for iplane in range(nplanes): msk,ctr = load_msk_ctr(datafiles[iplane]) cell_mask = np.concatenate((cell_mask,msk),axis=0) cell_center = np.concatenate((cell_center,ctr),axis=0) cell_depth = np.concatenate((cell_depth,iplane*np.ones((msk.shape[0],)))) # try: # try: # #if True: # #mean_image_red,mean_image_green = ut.loadmat(datafiles[0],['red_mean','green_mean']) # #mean_red_channel = np.zeros((len(datafiles),)+mean_image_red.shape) # #mean_green_channel = np.zeros((len(datafiles),)+mean_image_green.shape) # mean_red_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_green_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # for iplane in range(nplanes): # mean_image_red,mean_image_green = ut.loadmat(datafiles[iplane],['red_mean','green_mean']) # mean_red_channel[iplane] = mean_image_red # mean_green_channel[iplane] = mean_image_green # except: # mean_red_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_red_channel_corrected = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_green_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_green_channel_enhanced = np.zeros((nplanes,)+cell_mask.shape[1:]) # for iplane in range(nplanes): # mean_image_green,mean_image_green_enhanced,mean_image_red,mean_image_red_corrected = ut.loadmat(datafiles[iplane],['meanImg','meanImgE','meanImg_chan2','meanImg_chan2_corrected']) # #mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],['meanImg','meanImg_chan2_corrected']) # mean_red_channel[iplane] = mean_image_red # mean_red_channel_corrected[iplane] = mean_image_red_corrected # mean_green_channel[iplane] = mean_image_green # mean_green_channel_enhanced[iplane] = mean_image_green_enhanced # except: # print('no mean image data for ' + stimfile) # mean_red_channel = None # mean_red_channel_corrected = None # mean_green_channel = None # mean_green_channel_enhanced = None #varnames1 = ['green_mean','red_mean'] #varnames2= ['meanImg','meanImgE','meanImg_chan2','meanImg_chan2_corrected'] #outputs1 = ut.loadmat(datafiles[0],varnames1) #outputs2 = ut.loadmat(datafiles[0],varnames2) #use_first,use_second = [not outputs[0] is None for outputs in [outputs1,outputs2]] #if use_first: # mean_red_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_green_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # for iplane in range(nplanes): # mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],varnames1) # mean_green_channel[iplane] = mean_image_green # mean_red_channel[iplane] = mean_image_red # mean_red_channel_corrected = None # mean_green_channel_enhanced = None #elif use_second: # mean_green_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_green_channel_enhanced = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_red_channel = np.zeros((nplanes,)+cell_mask.shape[1:]) # mean_red_channel_corrected = np.zeros((nplanes,)+cell_mask.shape[1:]) # for iplane in range(nplanes): # mean_image_green,mean_image_green_enhanced,mean_image_red,mean_image_red_corrected = ut.loadmat(datafiles[iplane],['meanImg','meanImgE','meanImg_chan2','meanImg_chan2_corrected']) # #mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],['meanImg','meanImg_chan2_corrected']) # mean_red_channel[iplane] = mean_image_red # mean_red_channel_corrected[iplane] = mean_image_red_corrected # mean_green_channel[iplane] = mean_image_green # mean_green_channel_enhanced[iplane] = mean_image_green_enhanced #else: # print('no mean image data for ' + stimfile) # mean_red_channel = None # mean_red_channel_corrected = None # mean_green_channel = None # mean_green_channel_enhanced = None # trialize running and pupil data #try: roi_proc = load_roi_info(datafiles) #except: # roi_proc = None frame_div = np.floor(2*frame/nplanes).astype(np.int64) trialrun = ut.trialize(dxdt.T,frame,nbefore=nbefore,nafter=nafter) trialctr = ut.trialize(pupil_ctr,frame_div,nbefore=nbefore,nafter=nafter) trialfracctr = ut.trialize(pupil_frac_ctr,frame_div,nbefore=nbefore,nafter=nafter) trialarea = ut.trialize(pupil_area,frame_div,nbefore=nbefore,nafter=nafter) trialfracarea = ut.trialize(pupil_frac_area,frame_div,nbefore=nbefore,nafter=nafter) proc = {} proc['trialrun'] = trialrun proc['trialctr'] = trialctr proc['trialarea'] = trialarea proc['trialfracctr'] = trialfracctr proc['trialfracarea'] = trialfracarea proc['trialwise'] = trialwise proc['strialwise'] = strialwise proc['nbydepth'] = nbydepth proc['dtrialwise'] = dtrialwise proc['dfof'] = dfof proc['trialwise_t_offset'] = proc1['trialwise_t_offset'] proc['raw_trialwise'] = proc1['raw_trialwise'] proc['neuropil_trialwise'] = proc1['neuropil_trialwise'] if roi_proc: for key in roi_proc: proc[key] = roi_proc[key] else: print('could not compute roi info') #proc['mean_red_channel'] = mean_red_channel #proc['mean_red_channel_corrected'] = mean_red_channel_corrected #proc['mean_green_channel'] = mean_green_channel #proc['mean_green_channel_enhanced'] = mean_green_channel_enhanced #proc['cell_depth'] = cell_depth #proc['cell_center'] = cell_center #proc['cell_mask'] = cell_mask proc['nbefore'] = nbefore proc['nafter'] = nafter # define extra parameters based on 'result' variable for param in stim_params: name,function = param proc[name] = function(result) return proc
def load_msk_ctr(filename): msk,ctr = ut.loadmat(filename,['msk','ctr']) return msk.astype('bool').transpose((2,0,1)), ctr.T
def load_roi_info(datafiles): nplanes = len(datafiles) msk,ctr = load_msk_ctr(datafiles[0]) cell_mask = np.zeros((0,)+msk.shape[1:],dtype='bool') cell_center = np.zeros((0,2)) cell_depth = np.zeros((0,)) for iplane in range(nplanes): msk,ctr = load_msk_ctr(datafiles[iplane]) cell_mask = np.concatenate((cell_mask,msk),axis=0) cell_center = np.concatenate((cell_center,ctr),axis=0) cell_depth = np.concatenate((cell_depth,iplane*np.ones((msk.shape[0],)))) varnames1 = ['green_mean','red_mean'] varnames2= ['meanImg','meanImgE','meanImg_chan2','meanImg_chan2_corrected'] outputs1 = ut.loadmat(datafiles[0],varnames1) outputs2 = ut.loadmat(datafiles[0],varnames2) use_first,use_second = [not outputs[0] is None for outputs in [outputs1,outputs2]] if use_first: iplane = 0 mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],varnames1) shp = mean_image_green.shape mean_red_channel = np.zeros((nplanes,)+shp) mean_green_channel = np.zeros((nplanes,)+shp) for iplane in range(nplanes): mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],varnames1) mean_green_channel[iplane] = mean_image_green mean_red_channel[iplane] = mean_image_red mean_red_channel_corrected = None mean_green_channel_enhanced = None elif use_second: iplane = 0 mean_image_green,mean_image_green_enhanced,mean_image_red,mean_image_red_corrected = ut.loadmat(datafiles[iplane],varnames2) shp = mean_image_green.shape mean_green_channel = np.zeros((nplanes,)+shp) mean_green_channel_enhanced = np.zeros((nplanes,)+shp) mean_red_channel = np.zeros((nplanes,)+shp) mean_red_channel_corrected = np.zeros((nplanes,)+shp) for iplane in range(nplanes): mean_image_green,mean_image_green_enhanced,mean_image_red,mean_image_red_corrected = ut.loadmat(datafiles[iplane],varnames2) #mean_image_green,mean_image_red = ut.loadmat(datafiles[iplane],['meanImg','meanImg_chan2_corrected']) mean_red_channel[iplane] = mean_image_red mean_red_channel_corrected[iplane] = mean_image_red_corrected mean_green_channel[iplane] = mean_image_green mean_green_channel_enhanced[iplane] = mean_image_green_enhanced else: print('no mean image data for ' + datafiles[0]) mean_red_channel = None mean_red_channel_corrected = None mean_green_channel = None mean_green_channel_enhanced = None proc = {} proc['mean_red_channel'] = mean_red_channel proc['mean_red_channel_corrected'] = mean_red_channel_corrected proc['mean_green_channel'] = mean_green_channel proc['mean_green_channel_enhanced'] = mean_green_channel_enhanced proc['cell_depth'] = cell_depth proc['cell_center'] = cell_center proc['cell_mask'] = cell_mask return proc