def fill_in_default_metadata(rec, modelspecs, IsReload=False, **context):
'''
Sets any uninitialized metadata to defaults that should help us
find it in nems_db again. (fitter, recording, date, etc)
'''
if not IsReload:
# Add metadata to help you reload this state later
for modelspec in modelspecs:
meta = get_modelspec_metadata(modelspec)
if 'fitter' not in meta:
set_modelspec_metadata(modelspec, 'fitter', 'None')
if 'fit_time' not in meta:
set_modelspec_metadata(modelspec, 'fitter', 'None')
if 'recording' not in meta:
recname = rec.name if rec else 'None'
set_modelspec_metadata(modelspec, 'recording', recname)
if 'recording_uri' not in meta:
uri = rec.uri if rec and rec.uri else 'None'
set_modelspec_metadata(modelspec, 'recording_uri', uri)
if 'date' not in meta:
set_modelspec_metadata(modelspec, 'date', iso8601_datestring())
if 'hostname' not in meta:
set_modelspec_metadata(modelspec, 'hostname',
socket.gethostname())
return {'modelspecs': modelspecs}
def pup_pred_sum(batch=294, fs=4, jkn=20):
"""
# User parameters:
batch = 294 # VOC + pupil
#batch 289 # NAT + pupil
# fs = 4 # 20 Hz or 4 Hz
# jkn = 20
"""
if batch == 294:
modelnames = [
"psth.fs{}.pup-ld-st.pup_stategain.S_jk.nf{}-psthfr.j-basic".format(fs, jkn),
"psth.fs{}.pup-ld-st.pup0_stategain.S_jk.nf{}-psthfr.j-basic".format(fs, jkn)
]
elif batch == 289:
modelnames = [
"psth.fs{}.pup-ld-st.pup-hrc_stategain.S_jk.nf{}-psthfr.j-basic".format(fs, jkn),
"psth.fs{}.pup-ld-st.pup0-hrc_stategain.S_jk.nf{}-psthfr.j-basic".format(fs, jkn)
]
celldata = nd.get_batch_cells(batch=batch)
cellids = celldata['cellid'].tolist()
d = pd.DataFrame(columns=['cellid','state_chan','MI','MI_pup0','g','d',
'r','r_pup0','r_se','r_se_pup0'])
for mod_i, m in enumerate(modelnames):
print('Loading ', m)
modelspecs = nems_db.params._get_modelspecs(cellids, batch, m, multi='mean')
for modelspec in modelspecs:
c = modelspec[0]['meta']['cellid']
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
meta = ms.get_modelspec_metadata(modelspec)
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
sc = 'pupil'
j = 1
ii = ((d['cellid'] == c) & (d['state_chan'] == sc))
if np.sum(ii)==0:
d = d.append({'cellid': c, 'state_chan': sc}, ignore_index=True)
ii = ((d['cellid'] == c) & (d['state_chan'] == sc))
if mod_i == 0:
d.loc[ii, 'MI'] = (state_mod[j])
d.loc[ii, 'g'] = (gain[0,j])
d.loc[ii, 'd'] = (dc[0,j])
d.loc[ii, 'r'] = (meta['r_test'][0])
d.loc[ii, 'r_se'] = (meta['se_test'][0])
elif mod_i == 1:
d.loc[ii, 'MI_pup0'] = (state_mod[j])
d.loc[ii, 'r_pup0'] = (meta['r_test'][0])
d.loc[ii, 'r_se_pup0'] = (meta['se_test'][0])
d['goodcells'] = ((d['r']-d['r_se']) > (d['r_pup0']+d['r_se_pup0']))
#ax = None
#stateplots.beta_comp(d['r_pup0'], d['r'], n1="r pup0", n2="r pup",
# title='stategain', hist_range=[-0.05, 0.95],
# ax=ax, highlight=d['goodcells'])
return d
def tree_path(recording, modelspecs, xfspec):
'''
Returns a relative path (excluding filename, host, port) for URIs.
Editing this function edits the path in the file tree of every
file saved!
'''
xformname = xfspec_shortname(xfspec)
modelname = get_modelspec_shortname(modelspecs[0])
recname = recording.name # Or from rec.uri???
meta = get_modelspec_metadata(modelspecs[0])
date = meta.get('date', iso8601_datestring())
path = '/' + recname + '/' + modelname + '/' + xformname + '/' + date + '/'
return path
def get_model_results_per_state_model(batch=307,
state_list=None,
loader="psth.fs20.pup-ld-",
fitter="_jk.nf20-basic",
basemodel="-ref-psthfr.s_sdexp.S"):
"""
loader = "psth.fs20.pup-ld-"
fitter = "_jk.nf20-basic"
basemodel = "-ref-psthfr.s_sdexp.S"
state_list = ['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
d=get_model_results_per_state_model(batch=307, state_list=state_list,
loader=loader,fitter=fitter,
basemodel=basemodel)
state_list defaults to
['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
"""
if state_list is None:
state_list = [
'st.pup0.beh0', 'st.pup0.beh', 'st.pup.beh0', 'st.pup.beh'
]
modelnames = [loader + s + basemodel + fitter for s in state_list]
celldata = nd.get_batch_cells(batch=batch)
cellids = celldata['cellid'].tolist()
isolation = [
nd.get_isolation(cellid=c, batch=batch).loc[0, 'min_isolation']
for c in cellids
]
if state_list[-1].endswith('fil') or state_list[-1].endswith('pas'):
include_AP = True
else:
include_AP = False
d = pd.DataFrame(columns=[
'cellid', 'modelname', 'state_sig', 'state_chan', 'MI', 'isolation',
'r', 'r_se', 'd', 'g', 'sp', 'state_chan_alt'
])
new_sdexp = False
for mod_i, m in enumerate(modelnames):
print('Loading modelname: ', m)
modelspecs = nems_db.params._get_modelspecs(cellids,
batch,
m,
multi='mean')
for modelspec in modelspecs:
meta = ms.get_modelspec_metadata(modelspec)
phi = list(modelspec[0]['phi'].keys())
c = meta['cellid']
iso = isolation[cellids.index(c)]
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
if 'g' in phi:
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
elif ('amplitude_g' in phi) & ('amplitude_d' in phi):
new_sdexp = True
dc = None
gain = None
g_amplitude = modelspec[0]['phi']['amplitude_g']
g_base = modelspec[0]['phi']['base_g']
g_kappa = modelspec[0]['phi']['kappa_g']
g_offset = modelspec[0]['phi']['offset_g']
d_amplitude = modelspec[0]['phi']['amplitude_d']
d_base = modelspec[0]['phi']['base_d']
d_kappa = modelspec[0]['phi']['kappa_d']
d_offset = modelspec[0]['phi']['offset_d']
gain_mod = None
dc_mod = None
if 'state_mod_gain' in meta.keys():
gain_mod = meta['state_mod_gain']
dc_mod = meta['state_mod_dc']
if dc is not None:
sp = modelspec[0]['phi'].get('sp', np.zeros(gain.shape))
if dc.ndim > 1:
dc = dc[0, :]
gain = gain[0, :]
sp = sp[0, :]
a_count = 0
p_count = 0
for j, sc in enumerate(state_chans):
if gain is not None:
gain_val = gain[j]
dc_val = dc[j]
sp_val = sp[j]
else:
gain_val = None
dc_val = None
sp_val = None
r = {
'cellid': c,
'state_chan': sc,
'modelname': m,
'isolation': iso,
'state_sig': state_list[mod_i],
'g': gain_val,
'd': dc_val,
'sp': sp_val,
'MI': state_mod[j],
'r': meta['r_test'][0],
'r_se': meta['se_test'][0]
}
if new_sdexp:
r.update({
'g_amplitude': g_amplitude[0, j],
'g_base': g_base[0, j],
'g_kappa': g_kappa[0, j],
'g_offset': g_offset[0, j],
'd_amplitude': d_amplitude[0, j],
'd_base': d_base[0, j],
'd_kappa': d_kappa[0, j],
'd_offset': d_offset[0, j]
})
if gain_mod is not None:
r.update({'gain_mod': gain_mod[j], 'dc_mod': dc_mod[j]})
d = d.append(r, ignore_index=True)
l = len(d) - 1
if include_AP and sc.startswith("FILE_"):
siteid = c.split("-")[0]
fn = "%" + sc.replace("FILE_", "") + "%"
sql = "SELECT * FROM gDataRaw WHERE cellid=%s" +\
" AND parmfile like %s"
dcellfile = nd.pd_query(sql, (siteid, fn))
if dcellfile.loc[0]['behavior'] == 'active':
a_count += 1
d.loc[l,
'state_chan_alt'] = "ACTIVE_{}".format(a_count)
else:
p_count += 1
d.loc[l,
'state_chan_alt'] = "PASSIVE_{}".format(p_count)
else:
d.loc[l, 'state_chan_alt'] = d.loc[l, 'state_chan']
#d['r_unique'] = d['r'] - d['r0']
#d['MI_unique'] = d['MI'] - d['MI0']
return d
def get_model_results(batch=307,
state_list=None,
loader="psth.fs20.pup-ld-",
fitter="_jk.nf20-basic",
basemodel="-ref-psthfr.s_sdexp.S"):
"""
loader = "psth.fs20.pup-ld-"
fitter = "_jk.nf20-basic"
basemodel = "-ref-psthfr.s_sdexp.S"
state_list defaults to
['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
"""
if state_list is None:
state_list = [
'st.pup0.beh0', 'st.pup0.beh', 'st.pup.beh0', 'st.pup.beh'
]
modelnames = [loader + s + basemodel + fitter for s in state_list]
celldata = nd.get_batch_cells(batch=batch)
cellids = celldata['cellid'].tolist()
d = pd.DataFrame(columns=[
'cellid', 'modelname', 'state_sig', 'state_sig0', 'state_chan', 'MI',
'r', 'r_se', 'd', 'g', 'MI0', 'r0', 'r0_se'
])
for mod_i, m in enumerate(modelnames):
print('Loading modelname: ', m)
modelspecs = nems_db.params._get_modelspecs(cellids,
batch,
m,
multi='mean')
for modelspec in modelspecs:
meta = ms.get_modelspec_metadata(modelspec)
c = meta['cellid']
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
for j, sc in enumerate(state_chans):
ii = ((d['cellid'] == c) & (d['state_chan'] == sc))
if np.sum(ii) == 0:
r = {'cellid': c, 'state_chan': sc}
d = d.append(r, ignore_index=True)
ii = ((d['cellid'] == c) & (d['state_chan'] == sc))
if mod_i == 3:
# full model
d.loc[ii, ['modelname', 'state_sig', 'g', 'd', 'MI',
'r', 'r_se']] = \
[m, state_list[mod_i], gain[0, j], dc[0, j],
state_mod[j], meta['r_test'][0], meta['se_test'][0]]
elif (mod_i == 1) & (sc == 'pupil'):
# pupil shuffled model
d.loc[ii, ['state_sig0', 'MI0', 'r0', 'r0_se']] = \
[state_list[mod_i], state_mod[j],
meta['r_test'][0], meta['se_test'][0]]
elif (mod_i == 0) & (sc == 'baseline'):
d.loc[ii, ['state_sig0', 'r0', 'r0_se']] = \
[state_list[mod_i],
meta['r_test'][0], meta['se_test'][0]]
elif (mod_i == 2) & (sc not in ['baseline', 'pupil']):
# pupil shuffled model
d.loc[ii, ['state_sig0', 'MI0', 'r0', 'r0_se']] = \
[state_list[mod_i], state_mod[j],
meta['r_test'][0], meta['se_test'][0]]
d['r_unique'] = d['r'] - d['r0']
d['MI_unique'] = d['MI'] - d['MI0']
return d
d = pd.DataFrame(columns=[
'cellid', 'state_chan', 'MI', 'MI_fil0', 'MI_pup0', 'g', 'd', 'r',
'r_fil0', 'r_pup0', 'r_se', 'r_se_fil0', 'r_se_pup0'
])
for mod_i, m in enumerate(modelnames):
print('Loading ', m)
modelspecs = nems_db.params._get_modelspecs(cellids,
batch,
m,
multi='mean')
for modelspec in modelspecs:
c = modelspec[0]['meta']['cellid']
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
meta = ms.get_modelspec_metadata(modelspec)
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
for j, sc in enumerate(state_chans):
ii = ((d['cellid'] == c) & (d['state_chan'] == sc))
if np.sum(ii) == 0:
d = d.append(
{
'cellid': c,
'state_chan': sc,
'g': gain[0, j],
'd': dc[0, j],
'MI': state_mod[j]
},
ignore_index=True)
def get_model_results_per_state_model(batch=307,
state_list=None,
loader="psth.fs20.pup-ld-",
fitter="_jk.nf20-basic",
basemodel="-ref-psthfr.s_sdexp.S"):
"""
loader = "psth.fs20.pup-ld-"
fitter = "_jk.nf20-basic"
basemodel = "-ref-psthfr.s_sdexp.S"
state_list = ['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
d=get_model_results_per_state_model(batch=307, state_list=state_list,
loader=loader,fitter=fitter,
basemodel=basemodel)
state_list defaults to
['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
"""
if state_list is None:
state_list = [
'st.pup0.beh0', 'st.pup0.beh', 'st.pup.beh0', 'st.pup.beh'
]
modelnames = [loader + s + basemodel + fitter for s in state_list]
celldata = nd.get_batch_cells(batch=batch)
cellids = celldata['cellid'].tolist()
isolation = [
nd.get_isolation(cellid=c, batch=batch).loc[0, 'min_isolation']
for c in cellids
]
if state_list[-1].endswith('fil') or state_list[-1].endswith('pas'):
include_AP = True
else:
include_AP = False
DI_data = nd.pd_query(
"SELECT DISTINCT sCellFile.stimfile, sCellFile.cellid, sCellFile.rawid, gData.value" +\
" FROM gData INNER JOIN sCellFile ON gData.rawid=sCellFile.rawid" +\
" INNER JOIN sRunData ON sCellFile.cellid=sRunData.cellid" +\
f" WHERE sRunData.batch={batch} AND gData.name='DiscriminationIndex'" +\
" ORDER BY sCellFile.cellid, sCellFile.rawid")
u_cellids = list(set(DI_data['cellid']))
for c in u_cellids:
DI_cell = DI_data.loc[DI_data.cellid == c]
acount = 0
for index, row in DI_cell.iterrows():
acount += 1
astring = f"ACTIVE_{acount}"
DI_data.loc[(DI_data.cellid == c) &
(DI_data.rawid == row['rawid']),
'state_chan'] = astring
d = pd.DataFrame(columns=[
'cellid', 'modelname', 'state_sig', 'state_chan', 'MI', 'isolation',
'DI'
'r', 'r_se', 'd', 'g', 'sp', 'state_chan_alt'
])
new_sdexp = False
for mod_i, m in enumerate(modelnames):
print('Loading modelname: ', m)
modelspecs = nems_db.params._get_modelspecs(cellids,
batch,
m,
multi='mean')
for modelspec in modelspecs:
meta = ms.get_modelspec_metadata(modelspec)
phi = list(modelspec[0]['phi'].keys())
c = meta['cellid']
iso = isolation[cellids.index(c)]
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
if 'g' in phi:
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
elif ('amplitude_g' in phi) & ('amplitude_d' in phi):
new_sdexp = True
dc = None
gain = None
g_amplitude = modelspec[0]['phi']['amplitude_g']
g_base = modelspec[0]['phi']['base_g']
g_kappa = modelspec[0]['phi']['kappa_g']
g_offset = modelspec[0]['phi']['offset_g']
d_amplitude = modelspec[0]['phi']['amplitude_d']
d_base = modelspec[0]['phi']['base_d']
d_kappa = modelspec[0]['phi']['kappa_d']
d_offset = modelspec[0]['phi']['offset_d']
gain_mod = None
dc_mod = None
if 'state_mod_gain' in meta.keys():
gain_mod = meta['state_mod_gain']
dc_mod = meta['state_mod_dc']
if dc is not None:
sp = modelspec[0]['phi'].get('sp', np.zeros(gain.shape))
if dc.ndim > 1:
dc = dc[0, :]
gain = gain[0, :]
sp = sp[0, :]
a_count = 0
p_count = 0
if 'ACTIVE_0' in state_chans:
active_offset = 1
else:
active_offset = 0
for j, sc in enumerate(state_chans):
if gain is not None:
gain_val = gain[j]
dc_val = dc[j]
sp_val = sp[j]
else:
gain_val = None
dc_val = None
sp_val = None
#if c == 'BRT026c-02-2':
# import pdb
# pdb.set_trace()
if sc.startswith("ACTIVE"):
ac = int(sc.split("_")[-1]) + active_offset
sc_test = "ACTIVE_" + str(ac)
else:
sc_test = sc
v = DI_data.loc[(DI_data.cellid == c) &
(DI_data.state_chan == sc_test), 'value']
if len(v):
DI = v.values[0]
else:
DI = 0
r = {
'cellid': c,
'state_chan': sc,
'modelname': m,
'isolation': iso,
'DI': DI,
'state_sig': state_list[mod_i],
'g': gain_val,
'd': dc_val,
'sp': sp_val,
'MI': state_mod[j],
'r': meta['r_test'][0],
'r_se': meta['se_test'][0]
}
if new_sdexp:
r.update({
'g_amplitude': g_amplitude[0, j],
'g_base': g_base[0, j],
'g_kappa': g_kappa[0, j],
'g_offset': g_offset[0, j],
'd_amplitude': d_amplitude[0, j],
'd_base': d_base[0, j],
'd_kappa': d_kappa[0, j],
'd_offset': d_offset[0, j]
})
if gain_mod is not None:
r.update({'gain_mod': gain_mod[j], 'dc_mod': dc_mod[j]})
d = d.append(r, ignore_index=True)
l = len(d) - 1
if include_AP and sc.startswith("FILE_"):
siteid = c.split("-")[0]
fn = "%" + sc.replace("FILE_", "") + "%"
sql = "SELECT * FROM gDataRaw WHERE cellid=%s" +\
" AND parmfile like %s"
dcellfile = nd.pd_query(sql, (siteid, fn))
if dcellfile.loc[0]['behavior'] == 'active':
a_count += 1
d.loc[l,
'state_chan_alt'] = "ACTIVE_{}".format(a_count)
else:
p_count += 1
d.loc[l,
'state_chan_alt'] = "PASSIVE_{}".format(p_count)
else:
d.loc[l, 'state_chan_alt'] = d.loc[l, 'state_chan']
#d['r_unique'] = d['r'] - d['r0']
#d['MI_unique'] = d['MI'] - d['MI0']
return d