def add_ret_to_data_struct(filename, keylist=None, proc=None, grouplist=None):
with ut.hdf5edit(filename) as data_struct:
for key,group in zip(keylist,grouplist):
print((key,group))
print(list(proc[key].keys()))
if 'ret_vars' in proc[key]:
ret_vars = proc['/'.join([key,'ret_vars'])]
paramdict = proc['/'.join([key,'ret_vars','paramdict_normal'])]
rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],-paramdict['yo'][:][np.newaxis,:]),axis=0)
rf_sq_error = ret_vars['paramdict_normal']['sqerror'][:]
sx = ret_vars['paramdict_normal']['sigma_x'][:]
sy = ret_vars['paramdict_normal']['sigma_y'][:]
amp = ret_vars['paramdict_normal']['amplitude'][:]
this_expt = data_struct[group]
assign_(this_expt,'rf_mapping_pval',ret_vars['pval_ret'][:])
assign_(this_expt,'rf_ctr',rf_ctr)
assign_(this_expt,'rf_sq_error',rf_sq_error)
assign_(this_expt,'rf_sigma',np.sqrt(sx**2+sy**2))
assign_(this_expt,'rf_amplitude',amp)
#this_expt['rf_mapping_pval'] = ret_vars['pval_ret'][:]
# this_expt['rf_ctr'] = rf_ctr
# this_expt['rf_sq_error'] = rf_sq_error
# this_expt['rf_sigma'] = np.sqrt(sx**2+sy**2)
if 'position' in ret_vars:
stim_offset = ret_vars['position'][:] - paramdict['ctr'][:]
rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0))
rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis]
for kkey,vval in zip(['rf_distance_deg','rf_displacement_deg','stim_offset_deg'],[rf_distance_deg,rf_displacement_deg,stim_offset]):
assign_(this_expt,kkey,vval)
def add_data_struct_h5(filename,
cell_type='PyrL23',
keylist=None,
frame_rate_dict=None,
proc=None,
ret_vars=None,
nbefore=8,
nafter=8):
#with h5py.File(filename,mode='w+') as data_struct:
#with h5py.File(filename,mode='r') as data_struct:
with ut.hdf5edit(filename) as data_struct:
#data_struct = {}
for key in keylist:
if len(proc[key][0]) > 0:
gdind = 0
else:
gdind = 1
dfof = proc[key]['dtrialwise']
decon = proc[key]['strialwise']
running_speed_cm_s = 4 * np.pi / 180 * proc[key][
'trialrun'] # 4 cm from disk ctr to estimated mouse location
rf_ctr = np.concatenate(
(proc[key]['paramdict']['xo'][np.newaxis, :],
-proc[key]['paramdict']['yo'][np.newaxis, :]),
axis=0)
cell_id = np.arange(dfof.shape[0])
stimulus_id = proc[key]['locinds'].T - 1
session_id = 'session_' + key[:-1].replace('/', '_')
mouse_id = key.split('/')[1]
if not session_id in data_struct.keys():
this_session = data_struct.create_group(session_id)
this_session['mouse_id'] = mouse_id
this_session['cell_type'] = cell_type
this_session.create_dataset('cell_id', data=cell_id)
else:
this_session = data_struct[session_id]
exptno = 0
while 'retinotopy_' + str(exptno) in this_session.keys():
exptno = exptno + 1
this_expt = this_session.create_group('retinotopy_' + str(exptno))
this_expt.create_dataset('stimulus_id', data=stimulus_id)
this_expt['rf_mapping_pval'] = proc[key]['pval']
this_expt['rf_ctr'] = rf_ctr
this_expt.create_dataset('running_speed_cm_s',
data=running_speed_cm_s)
this_expt.create_dataset('F', data=dfof)
this_expt.create_dataset('raw_trialwise',
data=proc[key][gdind]['raw_trialwise'])
this_expt.create_dataset(
'neuropil_trialwise',
data=proc[key][gdind]['neuropil_trialwise'])
this_expt.create_dataset('decon', data=decon)
this_expt['nbefore'] = nbefore
this_expt['nafter'] = nafter
def add_evan_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, nbefore=8, nafter=8,featurenames=['size','contrast','angle'],datasetnames=None,groupname='size_contrast'):
if datasetnames is None:
datasetnames = ['stimulus_'+name for name in featurenames]
with ut.hdf5edit(filename) as data_struct:
grouplist = [None]*len(keylist)
for ikey,key in enumerate(keylist):
dfof = proc[key]['dtrialwise'][:]
decon = proc[key]['strialwise'][:]
trialwise = proc[key]['trialwise'][:]
trialrun = proc[key]['trialrun'][:] # 4 cm from disk ctr to estimated mouse location
ufeature_list,stimulus_id = gen_stimulus_id(proc[key],featurenames)
cell_id = np.arange(dfof.shape[0])
session_id = key
mouse_id = key.split('_')[1]
if not session_id in data_struct.keys():
this_session = data_struct.create_group(session_id)
this_session['mouse_id'] = mouse_id
this_session['cell_type'] = cell_type
this_session.create_dataset('cell_id',data=cell_id)
for field in ['cell_depth','cell_mask','cell_center','mean_red_channel','mean_green_channel']:
if field in proc[key]:
this_session.create_dataset(field,data=proc[key][field])
else:
this_session = data_struct[session_id]
exptno = 0
while groupname+'_'+str(exptno) in this_session.keys():
exptno = exptno+1
this_expt = this_session.create_group(groupname+'_'+str(exptno))
grouplist[ikey] = session_id + '/' + groupname + '_' + str(exptno)
this_expt.create_dataset('stimulus_id',data=stimulus_id)
stimulus_parameters = [n.encode('ascii','ignore') for n in datasetnames]
for name,ufeature in zip(datasetnames,ufeature_list):
this_expt.create_dataset(name,data=ufeature)
this_expt.create_dataset('trialrun',data=trialrun)
this_expt.create_dataset('F',data=dfof)
# this_expt.create_dataset('raw_trialwise',data=proc[key]['raw_trialwise'][:])
# this_expt.create_dataset('neuropil_trialwise',data=proc[key]['neuropil_trialwise'][:])
this_expt.create_dataset('decon',data=decon)
this_expt.create_dataset('nondecon',data=trialwise)
this_expt.create_dataset('stimulus_parameters',data=stimulus_parameters)
this_expt['nbefore'] = nbefore
this_expt['nafter'] = nafter
return grouplist
def add_data_struct_h5(filename,
cell_type='PyrL23',
keylist=None,
frame_rate_dict=None,
proc=None,
nbefore=8,
nafter=8):
#with h5py.File(filename,mode='w+') as data_struct:
with ut.hdf5edit(filename) as data_struct:
#data_struct = {}
ret_vars = {}
for key in keylist:
if len(proc[key]) > 0:
gdind = 0
else:
gdind = 1
dfof = proc[key]['dtrialwise'][:]
decon = proc[key]['strialwise'][:]
t_offset = proc[key]['trialwise_t_offset'][:]
ret_vars[key] = proc['/'.join([key, 'ret_vars'])]
#calcium_responses_au = np.nanmean(proc[key]['trialwise'][:,:,nbefore:-nafter],-1)
running_speed_cm_s = 4 * np.pi / 180 * proc[key][
'trialrun'][:] # 4 cm from disk ctr to estimated mouse location
paramdict = proc['/'.join([key, 'ret_vars', 'paramdict_normal'])]
#print(paramdict['xo'].shape)
rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis, :],
-paramdict['yo'][:][np.newaxis, :]),
axis=0)
stim_offset = ret_vars[key]['position'][:] - paramdict['ctr'][:]
rf_distance_deg = np.sqrt(
((rf_ctr - stim_offset[:, np.newaxis])**2).sum(0))
rf_displacement_deg = rf_ctr - stim_offset[:, np.newaxis]
cell_id = np.arange(dfof.shape[0])
unubs, inubs = np.unique(proc[key]['nubs'][:], return_inverse=True)
usize, isize = np.unique(proc[key]['size'][:], return_inverse=True)
uangle, iangle = np.unique(proc[key]['angle'][:],
return_inverse=True)
stimulus_id = np.concatenate(
(inubs, isize[np.newaxis], iangle[np.newaxis]), axis=0)
stimulus_nubs = unubs
stimulus_size_deg = usize
stimulus_direction = uangle
session_id = key
mouse_id = key.split('_')[1]
if not session_id in data_struct.keys():
this_session = data_struct.create_group(session_id)
this_session['mouse_id'] = mouse_id
this_session['cell_type'] = cell_type
this_session.create_dataset('cell_id', data=cell_id)
else:
this_session = data_struct[session_id]
exptno = 0
while 'nub_' + str(exptno) in this_session.keys():
exptno = exptno + 1
this_expt = this_session.create_group('nub_' + str(exptno))
this_expt.create_dataset('stimulus_id', data=stimulus_id)
this_expt.create_dataset('stimulus_nubs_active',
data=stimulus_size_deg)
this_expt.create_dataset('stimulus_size_deg',
data=stimulus_size_deg)
this_expt.create_dataset('stimulus_direction',
data=stimulus_direction)
this_expt['rf_mapping_pval'] = ret_vars[key]['pval_ret'][:]
this_expt['rf_distance_deg'] = rf_distance_deg
this_expt['rf_displacement_deg'] = rf_displacement_deg
this_expt['rf_ctr'] = rf_ctr
this_expt['stim_offset_deg'] = stim_offset
this_expt.create_dataset('running_speed_cm_s',
data=running_speed_cm_s)
this_expt.create_dataset('F', data=dfof)
this_expt.create_dataset('raw_trialwise',
data=proc[key]['raw_trialwise'][:])
this_expt.create_dataset('neuropil_trialwise',
data=proc[key]['neuropil_trialwise'][:])
this_expt.create_dataset('decon', data=decon)
this_expt.create_dataset('t_offset', data=t_offset)
this_expt['nbefore'] = nbefore
this_expt['nafter'] = nafter
def add_data_struct_h5(filename,
cell_type='PyrL23',
keylist=None,
frame_rate_dict=None,
proc=None,
ret_vars=None,
nbefore=8,
nafter=8):
#with h5py.File(filename,mode='w+') as data_struct:
with ut.hdf5edit(filename) as data_struct:
for key in keylist:
dfof = proc[key]['dtrialwise'][:]
decon = proc[key]['strialwise'][:]
running_speed_cm_s = 4 * np.pi / 180 * proc[key][
'trialrun'][:] # 4 cm from disk ctr to estimated mouse location
print(proc[key]['ret_vars']['paramdict_normal'])
ret_paramdict = proc['/'.join(
['', key, 'ret_vars', 'paramdict_normal'])]
xo = ret_paramdict['xo'][:]
yo = ret_paramdict['yo'][:]
rf_ctr = np.concatenate((xo[np.newaxis, :], -yo[np.newaxis, :]),
axis=0)
stim_offset = proc[key]['ret_vars']['position'][:] - ret_paramdict[
'ctr'][:]
rf_distance_deg = np.sqrt(
((rf_ctr - stim_offset[:, np.newaxis])**2).sum(0))
rf_displacement_deg = rf_ctr - stim_offset[:, np.newaxis]
cell_id = np.arange(dfof.shape[0])
uangle, iangle = np.unique(proc[key]['angle'][:],
return_inverse=True)
stimulus_id = np.zeros((2, ) +
proc[key]['paramdict']['iso'][:].shape)
#order = proc[key]['order']
for i, stimtype in enumerate(order):
stimulus_id[0][proc[key]['paramdict'][stimtype]] = i
stimulus_id[1] = proc[key]['angle']
stimulus_direction = uangle
session_id = key #'session_'+key[:-1].replace('/','_')
#mouse_id = key.split('/')[1]
mouse_id = key.split('_')[1]
if not session_id in data_struct.keys():
this_session = data_struct.create_group(session_id)
this_session['mouse_id'] = mouse_id
this_session['cell_type'] = cell_type
this_session.create_dataset('cell_id', data=cell_id)
else:
this_session = data_struct[session_id]
exptno = 0
while 'figure_ground_' + str(exptno) in this_session.keys():
exptno = exptno + 1
this_expt = this_session.create_group('figure_ground_' +
str(exptno))
this_expt.create_dataset('stimulus_id', data=stimulus_id)
#uorder = [u'ctrl',u'fig',u'grnd',u'iso',u'cross']
#this_expt.create_dataset('order',data=uorder)
this_expt.create_dataset('stimulus_direction',
data=stimulus_direction)
this_expt['stim_offset_deg'] = stim_offset
this_expt.create_dataset('running_speed_cm_s',
data=running_speed_cm_s)
this_expt.create_dataset('F', data=dfof)
this_expt.create_dataset('raw_trialwise',
data=proc[key]['raw_trialwise'])
this_expt.create_dataset('neuropil_trialwise',
data=proc[key]['neuropil_trialwise'])
this_expt.create_dataset('decon', data=decon)
this_expt['nbefore'] = nbefore
this_expt['nafter'] = nafter
def add_data_struct_h5(filename, cell_type='PyrL23', keylist=None, frame_rate_dict=None, proc=None, nbefore=8, nafter=8,featurenames=['size','contrast','angle'],datasetnames=None,groupname='size_contrast',replace=False):
if datasetnames is None:
datasetnames = ['stimulus_'+name for name in featurenames]
#with h5py.File(filename,mode='w+') as data_struct:
with ut.hdf5edit(filename) as data_struct:
#data_struct = {}
ret_vars = {}
grouplist = [None]*len(keylist)
for ikey,key in enumerate(keylist):
dfof = proc[key]['dtrialwise'][:]
decon = proc[key]['strialwise'][:]
t_offset = proc[key]['trialwise_t_offset'][:]
running_speed_cm_s = 4*np.pi/180*proc[key]['trialrun'][:] # 4 cm from disk ctr to estimated mouse location
#ucontrast,icontrast = np.unique(proc[key]['contrast'][:],return_inverse=True)
#usize,isize = np.unique(proc[key]['size'][:],return_inverse=True)
#uangle,iangle = np.unique(proc[key]['angle'][:],return_inverse=True)
#stimulus_id = np.concatenate((isize[np.newaxis],icontrast[np.newaxis],iangle[np.newaxis]),axis=0)
#stimulus_size_deg = usize
#stimulus_contrast = ucontrast
#stimulus_direction = uangle
ufeature_list,stimulus_id = gen_stimulus_id(proc[key],featurenames)
cell_id = np.arange(dfof.shape[0])
session_id = key
mouse_id = key.split('_')[1]
if session_id in data_struct.keys():
if len(cell_id) != len(data_struct[session_id]['cell_id']):
del data_struct[session_id]
if not session_id in data_struct.keys():
this_session = data_struct.create_group(session_id)
this_session['mouse_id'] = mouse_id
this_session['cell_type'] = cell_type
this_session.create_dataset('cell_id',data=cell_id)
for field in ['cell_depth','cell_mask','cell_center','mean_red_channel','mean_red_channel_corrected','mean_green_channel','mean_green_channel_enhanced']:
if field in proc[key]:
this_session.create_dataset(field,data=proc[key][field])
else:
this_session = data_struct[session_id]
if 'ret_vars' in proc[key]:
ret_vars[key] = proc['/'.join([key,'ret_vars'])]
paramdict = proc['/'.join([key,'ret_vars','paramdict_normal'])]
#rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],-paramdict['yo'][:][np.newaxis,:]),axis=0)
rf_ctr = np.concatenate((paramdict['xo'][:][np.newaxis,:],paramdict['yo'][:][np.newaxis,:]),axis=0)# new version uses y+ is up sign convention from the beginning
stim_offset = np.array((0,0)) #ret_vars[key]['position'][:] - paramdict['ctr'][:]
pval_ret = proc['/'.join([key,'ret_vars','pval_ret'])]
rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0))
rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis]
elif 'retinotopy_0' in this_session and 'rf_ctr' in this_session['retinotopy_0']:
rf_ctr = this_session['retinotopy_0']['rf_ctr'][:]
#ctr = this_session['retinotopy_0']['ctr'][:] # center of retinotopic mapping stimuli
stim_offset = proc[key]['position'][:]# - ctr # center of size contrast stimuli e.g. w/r/t retino center -- NOW WRT CENTER OF SCREEN
#stim_offset = proc[key]['position'][:][::-1]# - ctr # center of size contrast stimuli e.g. w/r/t retino center -- NOW WRT CENTER OF SCREEN; NOW IN Y,X ORDER, 21/4/22
pval_ret = this_session['retinotopy_0']['rf_mapping_pval'][:]
rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0))
rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis]
#elif 'retinotopy_0' in this_session and 'rf_ctr' in this_session['retinotopy_0']:
# rf_ctr = this_session['retinotopy_0']['rf_ctr'][:]*np.array((1,-1))[:,np.newaxis]
# ctr = this_session['retinotopy_0']['ctr'][:] # center of retinotopic mapping stimuli
# stim_offset = proc[key]['position'][:] - ctr # center of size contrast stimuli e.g. w/r/t retino center
# rf_distance_deg = np.sqrt(((rf_ctr-stim_offset[:,np.newaxis])**2).sum(0))
# rf_displacement_deg = rf_ctr-stim_offset[:,np.newaxis]
exptno = 0
if replace and groupname+'_0' in this_session.keys():
del this_session[groupname+'_0']
else:
while groupname+'_'+str(exptno) in this_session.keys():
exptno = exptno+1
this_expt = this_session.create_group(groupname+'_'+str(exptno))
grouplist[ikey] = session_id + '/' + groupname + '_' + str(exptno)
this_expt.create_dataset('stimulus_id',data=stimulus_id)
stimulus_parameters = [n.encode('ascii','ignore') for n in datasetnames]
for name,ufeature in zip(datasetnames,ufeature_list):
this_expt.create_dataset(name,data=ufeature)
#if 'ret_vars' in proc[key]:
# this_expt['rf_mapping_pval'] = ret_vars[key]['pval_ret'][:]
# this_expt['rf_distance_deg'] = rf_distance_deg
# this_expt['rf_displacement_deg'] = rf_displacement_deg
# this_expt['rf_ctr'] = rf_ctr
# this_expt['stim_offset_deg'] = stim_offset
if 'retinotopy_0' in this_session or 'ret_vars' in proc[key]:
this_expt['rf_mapping_pval'] = pval_ret[:]
this_expt['rf_distance_deg'] = rf_distance_deg
this_expt['rf_displacement_deg'] = rf_displacement_deg
this_expt['rf_ctr'] = rf_ctr
this_expt['stim_offset_deg'] = stim_offset
this_expt.create_dataset('running_speed_cm_s',data=running_speed_cm_s)
this_expt.create_dataset('F',data=dfof)
this_expt.create_dataset('raw_trialwise',data=proc[key]['raw_trialwise'][:])
this_expt.create_dataset('neuropil_trialwise',data=proc[key]['neuropil_trialwise'][:])
if 'trialctr' in proc[key]:
this_expt.create_dataset('pupil_ctr_trialwise_pix',data=proc[key]['trialctr'][:])
this_expt.create_dataset('pupil_ctr_trialwise_pct_eye_diam',data=proc[key]['trialfracctr'][:])
this_expt.create_dataset('pupil_area_trialwise_pix',data=proc[key]['trialarea'][:])
this_expt.create_dataset('pupil_area_trialwise_pct_eye_area',data=proc[key]['trialfracarea'][:])
this_expt.create_dataset('decon',data=decon)
this_expt.create_dataset('t_offset',data=t_offset)
this_expt.create_dataset('stimulus_parameters',data=stimulus_parameters)
this_expt['nbefore'] = nbefore
this_expt['nafter'] = nafter
return grouplist