def add_ret_to_data_struct(filename, keylist=None, proc=None, grouplist=None): with ut.hdf5edit(filename) as data_struct: for key,group in zip(keylist,grouplist): print((key,group)) print(list(proc[key].keys())) if 'ret_vars' in proc[key]: ret_vars = proc['/'.join([key,'ret_vars'])] paramdict = proc['/'.join([key,'ret_vars','paramdict_normal'])] rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],-paramdict['yo'][:][np.newaxis,:]),axis=0) rf_sq_error = ret_vars['paramdict_normal']['sqerror'][:] sx = ret_vars['paramdict_normal']['sigma_x'][:] sy = ret_vars['paramdict_normal']['sigma_y'][:] amp = ret_vars['paramdict_normal']['amplitude'][:] this_expt = data_struct[group] assign_(this_expt,'rf_mapping_pval',ret_vars['pval_ret'][:]) assign_(this_expt,'rf_ctr',rf_ctr) assign_(this_expt,'rf_sq_error',rf_sq_error) assign_(this_expt,'rf_sigma',np.sqrt(sx**2+sy**2)) assign_(this_expt,'rf_amplitude',amp) #this_expt['rf_mapping_pval'] = ret_vars['pval_ret'][:] # this_expt['rf_ctr'] = rf_ctr # this_expt['rf_sq_error'] = rf_sq_error # this_expt['rf_sigma'] = np.sqrt(sx**2+sy**2) if 'position' in ret_vars: stim_offset = ret_vars['position'][:] - paramdict['ctr'][:] rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0)) rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis] for kkey,vval in zip(['rf_distance_deg','rf_displacement_deg','stim_offset_deg'],[rf_distance_deg,rf_displacement_deg,stim_offset]): assign_(this_expt,kkey,vval)
def add_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, ret_vars=None, nbefore=8, nafter=8): #with h5py.File(filename,mode='w+') as data_struct: #with h5py.File(filename,mode='r') as data_struct: with ut.hdf5edit(filename) as data_struct: #data_struct = {} for key in keylist: if len(proc[key][0]) > 0: gdind = 0 else: gdind = 1 dfof = proc[key]['dtrialwise'] decon = proc[key]['strialwise'] running_speed_cm_s = 4 * np.pi / 180 * proc[key][ 'trialrun'] # 4 cm from disk ctr to estimated mouse location rf_ctr = np.concatenate( (proc[key]['paramdict']['xo'][np.newaxis, :], -proc[key]['paramdict']['yo'][np.newaxis, :]), axis=0) cell_id = np.arange(dfof.shape[0]) stimulus_id = proc[key]['locinds'].T - 1 session_id = 'session_' + key[:-1].replace('/', '_') mouse_id = key.split('/')[1] if not session_id in data_struct.keys(): this_session = data_struct.create_group(session_id) this_session['mouse_id'] = mouse_id this_session['cell_type'] = cell_type this_session.create_dataset('cell_id', data=cell_id) else: this_session = data_struct[session_id] exptno = 0 while 'retinotopy_' + str(exptno) in this_session.keys(): exptno = exptno + 1 this_expt = this_session.create_group('retinotopy_' + str(exptno)) this_expt.create_dataset('stimulus_id', data=stimulus_id) this_expt['rf_mapping_pval'] = proc[key]['pval'] this_expt['rf_ctr'] = rf_ctr this_expt.create_dataset('running_speed_cm_s', data=running_speed_cm_s) this_expt.create_dataset('F', data=dfof) this_expt.create_dataset('raw_trialwise', data=proc[key][gdind]['raw_trialwise']) this_expt.create_dataset( 'neuropil_trialwise', data=proc[key][gdind]['neuropil_trialwise']) this_expt.create_dataset('decon', data=decon) this_expt['nbefore'] = nbefore this_expt['nafter'] = nafter
def add_evan_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, nbefore=8, nafter=8,featurenames=['size','contrast','angle'],datasetnames=None,groupname='size_contrast'): if datasetnames is None: datasetnames = ['stimulus_'+name for name in featurenames] with ut.hdf5edit(filename) as data_struct: grouplist = [None]*len(keylist) for ikey,key in enumerate(keylist): dfof = proc[key]['dtrialwise'][:] decon = proc[key]['strialwise'][:] trialwise = proc[key]['trialwise'][:] trialrun = proc[key]['trialrun'][:] # 4 cm from disk ctr to estimated mouse location ufeature_list,stimulus_id = gen_stimulus_id(proc[key],featurenames) cell_id = np.arange(dfof.shape[0]) session_id = key mouse_id = key.split('_')[1] if not session_id in data_struct.keys(): this_session = data_struct.create_group(session_id) this_session['mouse_id'] = mouse_id this_session['cell_type'] = cell_type this_session.create_dataset('cell_id',data=cell_id) for field in ['cell_depth','cell_mask','cell_center','mean_red_channel','mean_green_channel']: if field in proc[key]: this_session.create_dataset(field,data=proc[key][field]) else: this_session = data_struct[session_id] exptno = 0 while groupname+'_'+str(exptno) in this_session.keys(): exptno = exptno+1 this_expt = this_session.create_group(groupname+'_'+str(exptno)) grouplist[ikey] = session_id + '/' + groupname + '_' + str(exptno) this_expt.create_dataset('stimulus_id',data=stimulus_id) stimulus_parameters = [n.encode('ascii','ignore') for n in datasetnames] for name,ufeature in zip(datasetnames,ufeature_list): this_expt.create_dataset(name,data=ufeature) this_expt.create_dataset('trialrun',data=trialrun) this_expt.create_dataset('F',data=dfof) # this_expt.create_dataset('raw_trialwise',data=proc[key]['raw_trialwise'][:]) # this_expt.create_dataset('neuropil_trialwise',data=proc[key]['neuropil_trialwise'][:]) this_expt.create_dataset('decon',data=decon) this_expt.create_dataset('nondecon',data=trialwise) this_expt.create_dataset('stimulus_parameters',data=stimulus_parameters) this_expt['nbefore'] = nbefore this_expt['nafter'] = nafter return grouplist
def add_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, nbefore=8, nafter=8): #with h5py.File(filename,mode='w+') as data_struct: with ut.hdf5edit(filename) as data_struct: #data_struct = {} ret_vars = {} for key in keylist: if len(proc[key]) > 0: gdind = 0 else: gdind = 1 dfof = proc[key]['dtrialwise'][:] decon = proc[key]['strialwise'][:] t_offset = proc[key]['trialwise_t_offset'][:] ret_vars[key] = proc['/'.join([key, 'ret_vars'])] #calcium_responses_au = np.nanmean(proc[key]['trialwise'][:,:,nbefore:-nafter],-1) running_speed_cm_s = 4 * np.pi / 180 * proc[key][ 'trialrun'][:] # 4 cm from disk ctr to estimated mouse location paramdict = proc['/'.join([key, 'ret_vars', 'paramdict_normal'])] #print(paramdict['xo'].shape) rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis, :], -paramdict['yo'][:][np.newaxis, :]), axis=0) stim_offset = ret_vars[key]['position'][:] - paramdict['ctr'][:] rf_distance_deg = np.sqrt( ((rf_ctr - stim_offset[:, np.newaxis])**2).sum(0)) rf_displacement_deg = rf_ctr - stim_offset[:, np.newaxis] cell_id = np.arange(dfof.shape[0]) unubs, inubs = np.unique(proc[key]['nubs'][:], return_inverse=True) usize, isize = np.unique(proc[key]['size'][:], return_inverse=True) uangle, iangle = np.unique(proc[key]['angle'][:], return_inverse=True) stimulus_id = np.concatenate( (inubs, isize[np.newaxis], iangle[np.newaxis]), axis=0) stimulus_nubs = unubs stimulus_size_deg = usize stimulus_direction = uangle session_id = key mouse_id = key.split('_')[1] if not session_id in data_struct.keys(): this_session = data_struct.create_group(session_id) this_session['mouse_id'] = mouse_id this_session['cell_type'] = cell_type this_session.create_dataset('cell_id', data=cell_id) else: this_session = data_struct[session_id] exptno = 0 while 'nub_' + str(exptno) in this_session.keys(): exptno = exptno + 1 this_expt = this_session.create_group('nub_' + str(exptno)) this_expt.create_dataset('stimulus_id', data=stimulus_id) this_expt.create_dataset('stimulus_nubs_active', data=stimulus_size_deg) this_expt.create_dataset('stimulus_size_deg', data=stimulus_size_deg) this_expt.create_dataset('stimulus_direction', data=stimulus_direction) this_expt['rf_mapping_pval'] = ret_vars[key]['pval_ret'][:] this_expt['rf_distance_deg'] = rf_distance_deg this_expt['rf_displacement_deg'] = rf_displacement_deg this_expt['rf_ctr'] = rf_ctr this_expt['stim_offset_deg'] = stim_offset this_expt.create_dataset('running_speed_cm_s', data=running_speed_cm_s) this_expt.create_dataset('F', data=dfof) this_expt.create_dataset('raw_trialwise', data=proc[key]['raw_trialwise'][:]) this_expt.create_dataset('neuropil_trialwise', data=proc[key]['neuropil_trialwise'][:]) this_expt.create_dataset('decon', data=decon) this_expt.create_dataset('t_offset', data=t_offset) this_expt['nbefore'] = nbefore this_expt['nafter'] = nafter
def add_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, ret_vars=None, nbefore=8, nafter=8): #with h5py.File(filename,mode='w+') as data_struct: with ut.hdf5edit(filename) as data_struct: for key in keylist: dfof = proc[key]['dtrialwise'][:] decon = proc[key]['strialwise'][:] running_speed_cm_s = 4 * np.pi / 180 * proc[key][ 'trialrun'][:] # 4 cm from disk ctr to estimated mouse location print(proc[key]['ret_vars']['paramdict_normal']) ret_paramdict = proc['/'.join( ['', key, 'ret_vars', 'paramdict_normal'])] xo = ret_paramdict['xo'][:] yo = ret_paramdict['yo'][:] rf_ctr = np.concatenate((xo[np.newaxis, :], -yo[np.newaxis, :]), axis=0) stim_offset = proc[key]['ret_vars']['position'][:] - ret_paramdict[ 'ctr'][:] rf_distance_deg = np.sqrt( ((rf_ctr - stim_offset[:, np.newaxis])**2).sum(0)) rf_displacement_deg = rf_ctr - stim_offset[:, np.newaxis] cell_id = np.arange(dfof.shape[0]) uangle, iangle = np.unique(proc[key]['angle'][:], return_inverse=True) stimulus_id = np.zeros((2, ) + proc[key]['paramdict']['iso'][:].shape) #order = proc[key]['order'] for i, stimtype in enumerate(order): stimulus_id[0][proc[key]['paramdict'][stimtype]] = i stimulus_id[1] = proc[key]['angle'] stimulus_direction = uangle session_id = key #'session_'+key[:-1].replace('/','_') #mouse_id = key.split('/')[1] mouse_id = key.split('_')[1] if not session_id in data_struct.keys(): this_session = data_struct.create_group(session_id) this_session['mouse_id'] = mouse_id this_session['cell_type'] = cell_type this_session.create_dataset('cell_id', data=cell_id) else: this_session = data_struct[session_id] exptno = 0 while 'figure_ground_' + str(exptno) in this_session.keys(): exptno = exptno + 1 this_expt = this_session.create_group('figure_ground_' + str(exptno)) this_expt.create_dataset('stimulus_id', data=stimulus_id) #uorder = [u'ctrl',u'fig',u'grnd',u'iso',u'cross'] #this_expt.create_dataset('order',data=uorder) this_expt.create_dataset('stimulus_direction', data=stimulus_direction) this_expt['stim_offset_deg'] = stim_offset this_expt.create_dataset('running_speed_cm_s', data=running_speed_cm_s) this_expt.create_dataset('F', data=dfof) this_expt.create_dataset('raw_trialwise', data=proc[key]['raw_trialwise']) this_expt.create_dataset('neuropil_trialwise', data=proc[key]['neuropil_trialwise']) this_expt.create_dataset('decon', data=decon) this_expt['nbefore'] = nbefore this_expt['nafter'] = nafter
def add_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, nbefore=8, nafter=8,featurenames=['size','contrast','angle'],datasetnames=None,groupname='size_contrast',replace=False): if datasetnames is None: datasetnames = ['stimulus_'+name for name in featurenames] #with h5py.File(filename,mode='w+') as data_struct: with ut.hdf5edit(filename) as data_struct: #data_struct = {} ret_vars = {} grouplist = [None]*len(keylist) for ikey,key in enumerate(keylist): dfof = proc[key]['dtrialwise'][:] decon = proc[key]['strialwise'][:] t_offset = proc[key]['trialwise_t_offset'][:] running_speed_cm_s = 4*np.pi/180*proc[key]['trialrun'][:] # 4 cm from disk ctr to estimated mouse location #ucontrast,icontrast = np.unique(proc[key]['contrast'][:],return_inverse=True) #usize,isize = np.unique(proc[key]['size'][:],return_inverse=True) #uangle,iangle = np.unique(proc[key]['angle'][:],return_inverse=True) #stimulus_id = np.concatenate((isize[np.newaxis],icontrast[np.newaxis],iangle[np.newaxis]),axis=0) #stimulus_size_deg = usize #stimulus_contrast = ucontrast #stimulus_direction = uangle ufeature_list,stimulus_id = gen_stimulus_id(proc[key],featurenames) cell_id = np.arange(dfof.shape[0]) session_id = key mouse_id = key.split('_')[1] if session_id in data_struct.keys(): if len(cell_id) != len(data_struct[session_id]['cell_id']): del data_struct[session_id] if not session_id in data_struct.keys(): this_session = data_struct.create_group(session_id) this_session['mouse_id'] = mouse_id this_session['cell_type'] = cell_type this_session.create_dataset('cell_id',data=cell_id) for field in ['cell_depth','cell_mask','cell_center','mean_red_channel','mean_red_channel_corrected','mean_green_channel','mean_green_channel_enhanced']: if field in proc[key]: this_session.create_dataset(field,data=proc[key][field]) else: this_session = data_struct[session_id] if 'ret_vars' in proc[key]: ret_vars[key] = proc['/'.join([key,'ret_vars'])] paramdict = proc['/'.join([key,'ret_vars','paramdict_normal'])] #rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],-paramdict['yo'][:][np.newaxis,:]),axis=0) rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],paramdict['yo'][:][np.newaxis,:]),axis=0)# new version uses y+ is up sign convention from the beginning stim_offset = np.array((0,0)) #ret_vars[key]['position'][:] - paramdict['ctr'][:] pval_ret = proc['/'.join([key,'ret_vars','pval_ret'])] rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0)) rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis] elif 'retinotopy_0' in this_session and 'rf_ctr' in this_session['retinotopy_0']: rf_ctr = this_session['retinotopy_0']['rf_ctr'][:] #ctr = this_session['retinotopy_0']['ctr'][:] # center of retinotopic mapping stimuli stim_offset = proc[key]['position'][:]# - ctr # center of size contrast stimuli e.g. w/r/t retino center -- NOW WRT CENTER OF SCREEN #stim_offset = proc[key]['position'][:][::-1]# - ctr # center of size contrast stimuli e.g. w/r/t retino center -- NOW WRT CENTER OF SCREEN; NOW IN Y,X ORDER, 21/4/22 pval_ret = this_session['retinotopy_0']['rf_mapping_pval'][:] rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0)) rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis] #elif 'retinotopy_0' in this_session and 'rf_ctr' in this_session['retinotopy_0']: # rf_ctr = this_session['retinotopy_0']['rf_ctr'][:]*np.array((1,-1))[:,np.newaxis] # ctr = this_session['retinotopy_0']['ctr'][:] # center of retinotopic mapping stimuli # stim_offset = proc[key]['position'][:] - ctr # center of size contrast stimuli e.g. w/r/t retino center # rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0)) # rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis] exptno = 0 if replace and groupname+'_0' in this_session.keys(): del this_session[groupname+'_0'] else: while groupname+'_'+str(exptno) in this_session.keys(): exptno = exptno+1 this_expt = this_session.create_group(groupname+'_'+str(exptno)) grouplist[ikey] = session_id + '/' + groupname + '_' + str(exptno) this_expt.create_dataset('stimulus_id',data=stimulus_id) stimulus_parameters = [n.encode('ascii','ignore') for n in datasetnames] for name,ufeature in zip(datasetnames,ufeature_list): this_expt.create_dataset(name,data=ufeature) #if 'ret_vars' in proc[key]: # this_expt['rf_mapping_pval'] = ret_vars[key]['pval_ret'][:] # this_expt['rf_distance_deg'] = rf_distance_deg # this_expt['rf_displacement_deg'] = rf_displacement_deg # this_expt['rf_ctr'] = rf_ctr # this_expt['stim_offset_deg'] = stim_offset if 'retinotopy_0' in this_session or 'ret_vars' in proc[key]: this_expt['rf_mapping_pval'] = pval_ret[:] this_expt['rf_distance_deg'] = rf_distance_deg this_expt['rf_displacement_deg'] = rf_displacement_deg this_expt['rf_ctr'] = rf_ctr this_expt['stim_offset_deg'] = stim_offset this_expt.create_dataset('running_speed_cm_s',data=running_speed_cm_s) this_expt.create_dataset('F',data=dfof) this_expt.create_dataset('raw_trialwise',data=proc[key]['raw_trialwise'][:]) this_expt.create_dataset('neuropil_trialwise',data=proc[key]['neuropil_trialwise'][:]) if 'trialctr' in proc[key]: this_expt.create_dataset('pupil_ctr_trialwise_pix',data=proc[key]['trialctr'][:]) this_expt.create_dataset('pupil_ctr_trialwise_pct_eye_diam',data=proc[key]['trialfracctr'][:]) this_expt.create_dataset('pupil_area_trialwise_pix',data=proc[key]['trialarea'][:]) this_expt.create_dataset('pupil_area_trialwise_pct_eye_area',data=proc[key]['trialfracarea'][:]) this_expt.create_dataset('decon',data=decon) this_expt.create_dataset('t_offset',data=t_offset) this_expt.create_dataset('stimulus_parameters',data=stimulus_parameters) this_expt['nbefore'] = nbefore this_expt['nafter'] = nafter return grouplist