def refresh_batches(self):
sql = "SELECT * FROM Analysis order by id"
self.analysis_data = nd.pd_query(sql)
model = QtGui.QStandardItemModel()
for i in self.analysis_data['name'].to_list():
item = QtGui.QStandardItem(i)
model.appendRow(item)
self.comboAnalysis.setModel(model)
index = self.comboAnalysis.findText(self.current_analysis,
QtCore.Qt.MatchFixedString)
if index >= 0:
self.comboAnalysis.setCurrentIndex(index)
sql = "SELECT DISTINCT batch FROM Batches order by batch"
self.batch_data = nd.pd_query(sql)
model = QtGui.QStandardItemModel()
for i in self.batch_data['batch'].to_list():
item = QtGui.QStandardItem(i)
model.appendRow(item)
self.comboBatch.setModel(model)
self.analysis_update()
def update_widgets(self):
batch = int(self.batchLE.text())
cellmask = self.cellLE.text() + "%"
# strip white spaces and split by semicolon
modeltext = self.modelLE.text().replace(' ', '').split(';')
# attach % wildcard before and after modelstring
modelmask = ['%' + m + '%' for m in modeltext]
#modelmask = "%" + self.modelLE.text() + "%"
save_settings(self)
if batch > 0:
self.batch = batch
else:
self.batchLE.setText(str(self.batch))
#" ORDER BY cellid",
#self.d_cells = nd.get_batch_cells(self.batch, cellid=cellmask)
self.d_cells = nd.pd_query("SELECT DISTINCT cellid FROM Results" +
" WHERE batch=%s AND cellid like %s" +
" ORDER BY cellid",
(self.batch, cellmask))
modelquery = ("SELECT modelname, count(*) as n, max(lastmod) as "
"last_mod FROM Results WHERE batch=%s AND ")
for i, m in enumerate(modelmask):
modelquery += 'modelname like %s OR '
modelquery = modelquery[:-3] # drop the trailing OR
modelquery += 'GROUP BY modelname ORDER BY modelname'
self.d_models = nd.pd_query(modelquery, (self.batch, *modelmask))
# self.d_models = nd.pd_query("SELECT modelname, count(*) as n, max(lastmod) as last_mod FROM Results" +
# " WHERE batch=%s AND modelname like %s" +
# " GROUP BY modelname ORDER BY modelname",
# (self.batch, modelmask))
self.cells.clear()
for c in list(self.d_cells['cellid']):
list_item = qw.QListWidgetItem(c, self.cells)
self.models.clear()
for m in list(self.d_models['modelname']):
list_item = qw.QListWidgetItem(m, self.models)
#self.data_model._df = self.d_models
#self.data_table.setModel(self.data_model)
print('updated list widgets')
def save_analysis(self):
video_name = self.video_name.get()
fn = video_name + '.pickle'
fn_mat = video_name + '.mat'
fp = os.path.split(self.processed_video)[0]
save_path = os.path.join(fp, fn)
# for matlab loading
mat_fn = os.path.join(fp, fn_mat)
sorted_dir = os.path.split(save_path)[0]
if os.path.isdir(sorted_dir) != True:
# create sorted directory and force to be world writeable
os.system("mkdir {}".format(sorted_dir))
os.system("chmod a+w {}".format(sorted_dir))
print("created new directory {0}".format(sorted_dir))
else:
pass
save_dict = self.parms
# add excluded frames to the save dictionary
excluded_frames = np.concatenate(
(np.array(self.exclude_starts)[np.newaxis, :],
np.array(self.exclude_ends)[np.newaxis, :]),
axis=0)
save_dict['cnn']['excluded_frames'] = excluded_frames.T
print("computing eyespeed")
x_diff = np.diff(save_dict['cnn']['x'])
y_diff = np.diff(save_dict['cnn']['y'])
d = np.sqrt((x_diff**2) + (y_diff**2))
d[-1] = 0
d = np.concatenate((d, np.zeros(1)))
save_dict['cnn']['eyespeed'] = d
with open(save_path, 'wb') as fp:
pickle.dump(save_dict, fp, protocol=pickle.HIGHEST_PROTOCOL)
scipy.io.savemat(mat_fn, save_dict)
# finally, update celldb to mark pupil as analyzed
# see if the eyecalfile is correct (migth've been a flush error, in which case it'll still say L:/)
get_file1 = "SELECT eyecalfile from gDataRaw where eyecalfile='{0}'".format(
self.raw_video)
out1 = nd.pd_query(get_file1)
if out1.shape[0] == 0:
# try the L:/ path
og_video_path = self.raw_video.replace('/auto/data/daq/', 'L:/')
sql = "UPDATE gDataRaw SET eyewin=2 WHERE eyecalfile='{}'".format(
og_video_path)
else:
sql = "UPDATE gDataRaw SET eyewin=2 WHERE eyecalfile='{}'".format(
self.raw_video)
nd.sql_command(sql)
print("saved analysis successfully")
def get_site_data(siteid):
"""
:param siteid:
:return:
"""
sql = f"SELECT * FROM gCellMaster WHERE siteid like '{siteid}'"
d = db.pd_query(sql)
return d
def get_single_cell_data(cellid):
"""
:param cellid: single cellid or siteid
:return:
"""
sql = f"SELECT * FROM gSingleCell WHERE cellid like '{cellid}%%'"
d = db.pd_query(sql)
return d
def load_file(self):
"""
Load the overall predictions and plot the trace on the trace canvas.
Display the first frame of the video on the pupil canvas.
"""
params_file = self.video_name.get()
# get raw video -- try to use the exisiting path from raw video
fp = os.path.split(self.raw_video)[0]
self.processed_video = os.path.join(fp, 'sorted', params_file)
# reset raw video attribute
ext = self.raw_video.split('.')[-1]
if len(self.raw_video.split('.')) > 2:
ext2 = self.raw_video.split('.')[-2]
self.raw_video = os.path.join(fp,
params_file) + '.' + ext2 + '.' + ext
else:
self.raw_video = os.path.join(fp, params_file) + '.' + ext
print(self.raw_video)
self.plot_trace(params_file)
try:
self.plot_eyelid_movement(params_file)
except:
print("Couldn't load eyelid keypoints -- old fit?")
pass
self.frame_n_value.insert(0, str(0))
# reset exclusion frames
self.exclude_starts = []
self.exclude_ends = []
# save first ten frames and display the first
video = self.raw_video
# first make sure the tmp file doesn't exist for this user, just to avoid asking for overwrite permissions
os.system(f"rm {tmp_frame_folder}frame1_{getpass.getuser()}.jpg")
os.system(
f"ffmpeg -ss 00:00:00 -i {video} -vframes 1 {tmp_frame_folder}frame1_{getpass.getuser()}.jpg"
)
frame_file = tmp_frame_folder + f'frame1_{getpass.getuser()}.jpg'
# define the species for this animal by querying the database
self.species = nd.pd_query(
f"SELECT species from gAnimal where animal='{self.animal_name.get()}'"
).values[0][0]
self.plot_frame(frame_file)
self.master.mainloop()
def _queue_fits(batch, modelnames, iterator):
for siteid in iterator:
for modelname in modelnames:
do_fit = True
if not FORCE_RERUN:
d = nd.pd_query(
"SELECT * FROM Results WHERE cellid like %s and modelname=%s and batch=%s",
params=(siteid + "%", modelname, batch))
if len(d) > 0:
do_fit = False
print(f'Fit exists for {siteid} {batch} {modelname}')
if do_fit:
enqueue_exacloud_models(cellist=[siteid],
batch=batch,
modellist=[modelname],
useGPU=True,
**EXACLOUD_SETTINGS)
def list_all_recordings(self):
print('listing all taggable sites...')
df = nd.pd_query(
"SELECT * FROM gSingleCell INNER JOIN sCellFile ON gSingleCell.cellid = sCellFile.cellid"
+ " WHERE sCellFile.RunClassid = 51 AND sCellFile.cellid LIKE %s",
params=("TNC%", ))
# clean up DF
DF = pd.DataFrame()
DF['cellid'] = df.cellid.iloc[:, 0]
DF['siteid'] = df.siteid
DF['recording'] = df.stimfile.apply(lambda x: x.split('.')[0])
DF['parmfile'] = df.stimpath + df.stimfile # full path to parameter file.
DF['rawid'] = df.rawid.iloc[:, 1]
self.recordings = DF.recording.unique()
self.DF = DF
print('done')
def load_existing_pred(cellid=None,
siteid=None,
batch=None,
modelname_existing=None,
**kwargs):
"""
designed to be called by xforms keyword loadpred
cellid/siteid - one or the other required
batch - required
default modelname_existing = "psth.fs4.pup-ld-st.pup-hrc-psthfr-aev_sdexp2.SxR_newtf.n.lr1e4.cont.et5.i50000"
makes new signal 'pred0' from evaluated 'pred', returns in updated rec
returns ctx-compatible dict {'rec': nems.Recording, 'input_name': 'pred0'}
"""
if (batch is None):
raise ValueError("must specify cellid/siteid and batch")
if cellid is None:
if siteid is None:
raise ValueError("must specify cellid/siteid and batch")
d = nd.pd_query(
"SELECT batch,cellid FROM Batches WHERE batch=%s AND cellid like %s",
(
batch,
siteid + "%",
))
cellid = d['cellid'].values[0]
elif type(cellid) is list:
cellid = cellid[0]
if modelname_existing is None:
#modelname_existing = "psth.fs4.pup-ld-st.pup-hrc-psthfr-aev_sdexp2.SxR_newtf.n.lr1e4.cont"
modelname_existing = "psth.fs4.pup-ld-st.pup-hrc-psthfr-aev_sdexp2.SxR_newtf.n.lr1e4.cont.et5.i50000"
xf, ctx = xhelp.load_model_xform(cellid, batch, modelname_existing)
for k in ctx['val'].signals.keys():
if k not in ctx['rec'].signals.keys():
ctx['rec'].signals[k] = ctx['val'].signals[k].copy()
s = ctx['rec']['pred'].copy()
s.name = 'pred0'
ctx['rec'].add_signal(s)
#return {'rec': ctx['rec'],'val': ctx['val'],'est': ctx['est']}
return {'rec': ctx['rec'], 'input_name': 'pred0'}
def reload_models(self):
from nems_web.utilities.ModelFinder import ModelFinder
t = self.comboBatch.currentText()
session = Session()
Analysis = Tables()['Analysis']
modeltree = (session.query(Analysis.modeltree).filter(
Analysis.name == self.current_analysis).first())
modelextras = (session.query(Analysis.model_extras).filter(
Analysis.name == self.current_analysis).first())
# Pass modeltree string from Analysis to a ModelFinder constructor,
# which will use a series of internal methods to convert the tree string
# to a list of model names.
# Then add any additional models specified in extraModels, and add
# model_lists from extraAnalyses.
if modeltree and modeltree[0]:
#model_list = _get_models(modeltree[0])
load, mod, fit = json.loads(modeltree[0])
loader = ModelFinder(load).modellist
model = ModelFinder(mod).modellist
fitter = ModelFinder(fit).modellist
combined = itertools.product(loader, model, fitter)
model_list = ['_'.join(m) for m in combined]
extraModels = [
s.strip("\"\n").replace("\\n", "")
for s in modelextras[0].split(',')
]
model_list.extend(extraModels)
else:
model_list = []
self.all_models = model_list
#sql = f"SELECT DISTINCT modelname FROM Results WHERE batch={t}"
#data = nd.pd_query(sql)
#self.all_models = data['modelname'].to_list()
sql = f"SELECT DISTINCT cellid FROM Batches WHERE batch={t}"
data = nd.pd_query(sql)
self.all_cellids = data['cellid'].to_list()
self.lastbatch = t
self.labelBatchName.setText(str(t))
def second_fit_pop_models(batch, start_from=None, test_count=None):
all_cellids = nd.get_batch_cells(batch, as_list=True)
if batch == 322:
sites = NAT4_A1_SITES
else:
sites = NAT4_PEG_SITES
cellids = [
c for c in all_cellids
if np.any([c.startswith(s.split('.')[0]) for s in sites])
]
modelnames = []
for k, v in MODELGROUPS.items():
if ('_single' not in k) and ('_exploration' not in k) and (k != 'LN'):
modelnames.extend(v)
iterator = cellids
for siteid in iterator:
for modelname in modelnames[start_from:test_count]:
do_fit = True
if not FORCE_RERUN:
d = nd.pd_query(
"SELECT * FROM Results WHERE cellid like %s and modelname=%s and batch=%s",
params=(siteid + "%", modelname, batch))
if len(d) > 0:
do_fit = False
print(f'Fit exists for {siteid} {batch} {modelname}')
if do_fit:
nd.enqueue_models(
celllist=[siteid],
batch=batch,
modellist=[modelname],
user="******",
#executable_path='/auto/users/jacob/bin/anaconda3/envs/jacob_nems/bin/python',
executable_path=
'/auto/users/svd/bin/miniconda3/envs/tf/bin/python',
script_path=
'/auto/users/jacob/bin/anaconda3/envs/jacob_nems/nems/scripts/fit_single.py'
)
return modelnames
def get_training_files(animal, runclass, earliest_date, latest_date=None, pupil=False, min_trials=50):
an_regex = "%" + animal + "%"
if latest_date is None:
latest_date = earliest_date
# get list of all training parmfiles
sql = "SELECT parmfile, resppath FROM gDataRaw WHERE runclass=%s and resppath like %s and training = 1 and bad=0 and trials>%s"
if pupil:
# require pupil processed
sql = "SELECT parmfile, resppath FROM gDataRaw WHERE runclass=%s and resppath like %s and training = 1 and bad=0 and trials>%s and eyewin=2"
parmfiles = nd.pd_query(sql, (runclass, an_regex, min_trials))
try:
parmfiles['date'] = [dt.datetime.strptime('-'.join(x.split('_')[1:-2]), '%Y-%m-%d') for x in parmfiles.parmfile]
ed = dt.datetime.strptime(earliest_date, '%Y_%m_%d')
ld = dt.datetime.strptime(latest_date, '%Y_%m_%d')
parmfiles = parmfiles[(parmfiles.date >= ed) & (parmfiles.date <= ld)]
return parmfiles
except:
raise ValueError("No files found")
"""
Load spike waveform params for all SUs in batch 289 and 294. Cluster based on this.
"""
import nems.db as nd
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
path = '/auto/users/hellerc/results/nat_pupil_ms/'
cellids_cache = path + 'celltypes.csv'
cellids = pd.DataFrame(pd.concat([nd.get_batch_cells(289), nd.get_batch_cells(294), nd.get_batch_cells(331)]).cellid)
iso_query = f"SELECT cellid, rawid, isolation from gSingleRaw WHERE cellid in {tuple([x for x in cellids.cellid])}"
isolation = nd.pd_query(iso_query)
cellids = pd.DataFrame(data=isolation[isolation.isolation>=95].cellid.unique(), columns=['cellid'])
# keep only SU
sw = [nd.get_gSingleCell_meta(cellid=c, fields='wft_spike_width') for c in cellids.cellid]
cellids['spike_width'] = sw
# remove cellids that weren't sorted with KS (so don't have waveform stats)
cellids = cellids[cellids.spike_width!=-1]
# now save endslope and peak trough ratio
es = [nd.get_gSingleCell_meta(cellid=c, fields='wft_endslope') for c in cellids.cellid]
pt = [nd.get_gSingleCell_meta(cellid=c, fields='wft_peak_trough_ratio') for c in cellids.cellid]
cellids['end_slope'] = es
cellids['peak_trough'] = pt
def initialize_with_prefit(modelspec, meta, area="A1", cellid=None, siteid=None, batch=322, pre_batch=None,
use_matched=False, use_simulated=False, use_full_model=False,
prefit_type=None, freeze_early=True, IsReload=False, **ctx):
"""
replace early layers of model with fit parameters from a "standard" model ... for now that's model with the same architecture fit
to the NAT4 dataset
for dnn single:
initial model:
modelname = "ozgf.fs100.ch18-ld-norm.l1-sev_wc.18x4.g-fir.1x25x4-relu.4.f-wc.4x1-lvl.1-dexp.1_tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4.es20"
use initial as pre-fit:
modelname = "ozgf.fs100.ch18-ld-norm.l1-sev_wc.18x4.g-fir.1x25x4-relu.4.f-wc.4x1-lvl.1-dexp.1_prefit-tfinit.n.lr1e3.et3.es20-newtf.n.lr1e4.es20"
"""
if IsReload:
return {}
xi = find_module("weight_channels", modelspec, find_all_matches=True)
if len(xi) == 0:
raise ValueError(f"modelspec has not weight_channels layer to align")
copy_layers = xi[-1]
freeze_layer_count = xi[-1]
batch = int(meta['batch'])
modelname_parts = meta['modelname'].split("_")
if use_simulated:
guess = '.'.join(['SIM000a', modelname_parts[1]])
# remove problematic characters
guess = re.sub('[:]', '', guess)
guess = re.sub('[,]', '', guess)
if len(guess) > 100:
# If modelname is too long, causes filesystem errors.
guess = guess[:75] + '...' + str(hashlib.sha1(guess.encode('utf-8')).hexdigest()[:20])
old_uri = f"/auto/data/nems_db/modelspecs/{guess}/modelspec.0000.json"
log.info('loading saved modelspec from: ' + old_uri)
new_ctx = load_phi(modelspec, prefit_uri=old_uri, copy_layers=copy_layers)
return new_ctx
elif prefit_type == 'init':
# use full pop file - SVD work in progress. current best?
load_string_pop = "ozgf.fs100.ch18.pop-loadpop-norm.l1-popev"
fit_string_pop = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4"
pre_part = load_string_pop
if len(modelname_parts[2].split("-")) > 2:
post_part = "-".join(modelname_parts[2].split("-")[1:-1])
else:
post_part = "-".join(modelname_parts[2].split("-")[1:])
model_search = "_".join([pre_part, modelname_parts[1], post_part])
if pre_batch is None:
pre_batch = batch
if pre_batch in [322, 334]:
pre_cellid = 'ARM029a-07-6'
elif pre_batch == 323:
pre_cellid = 'ARM017a-01-9'
else:
raise ValueError(f"batch {pre_batch} prefit not implemented yet.")
log.info(f"prefit cellid={pre_cellid}, skipping init_fit")
copy_layers = len(modelspec)
elif use_full_model:
# use full pop file - SVD work in progress. current best?
load_string_pop = "ozgf.fs100.ch18.pop-loadpop-norm.l1-popev"
fit_string_pop = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4"
if prefit_type == 'heldout':
pre_part = "ozgf.fs100.ch18.pop-loadpop.hs-norm.l1-popev"
elif prefit_type == 'matched':
pre_part = "ozgf.fs100.ch18.pop-loadpop.hm-norm.l1-popev"
elif prefit_type == 'matched_half':
# 50% est data (matched cell excluded)
pre_part = "ozgf.fs100.ch18.pop-loadpop.hm-norm.l1-popev.k50"
elif prefit_type == 'matched_quarter':
# 25% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hm-norm.l1-popev.k25"
elif prefit_type == 'matched_fifteen':
# 15% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hm-norm.l1-popev.k15"
elif prefit_type == 'matched_ten':
# 10% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hm-norm.l1-popev.k10"
elif prefit_type == 'heldout_half':
# 50% est data, cell excluded (is this a useful condition?)
pre_part = "ozgf.fs100.ch18.pop-loadpop.hs-norm.l1-popev.k50"
elif prefit_type == 'heldout_quarter':
# 25% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hs-norm.l1-popev.k25"
elif prefit_type == 'heldout_fifteen':
# 15% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hs-norm.l1-popev.k15"
elif prefit_type == 'heldout_ten':
# 10% est data
pre_part = "ozgf.fs100.ch18.pop-loadpop.hs-norm.l1-popev.k10"
elif 'R.q.s' in modelname_parts[1]:
pre_part = "ozgf.fs100.ch18-ld-norm.l1-sev"
elif 'ch32' in modelname_parts[0]:
pre_part = "ozgf.fs100.ch32.pop-loadpop-norm.l1-popev"
elif 'ch64' in modelname_parts[0]:
pre_part = "ozgf.fs100.ch64.pop-loadpop-norm.l1-popev"
elif batch==333:
# not pre-concatenated recording. different stim for each site,
# so fit each site separately (unless titan)
pre_part = "ozgf.fs100.ch18-ld-norm.l1-sev"
else:
#load_string_pop = "ozgf.fs100.ch18.pop-loadpop-norm.l1-popev"
pre_part = load_string_pop
if prefit_type == 'titan':
if batch==333:
pre_part = load_string_pop
post_part = "tfinit.n.mc50.lr1e3.et4.es20-newtf.n.mc100.lr1e4"
else:
post_part = "tfinit.n.mc25.lr1e3.es20-newtf.n.mc100.lr1e4.exa"
else:
#post_part = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4.es20"
post_part = fit_string_pop
if modelname_parts[2].endswith(".l2:5") or modelname_parts[2].endswith(".l2:5-dstrf") or modelname_parts[2].endswith("ver5"):
post_part += ".l2:5"
elif modelname_parts[2].endswith(".l2:4") or modelname_parts[2].endswith(".l2:4-dstrf") or modelname_parts[2].endswith("ver4"):
post_part += ".l2:4"
elif modelname_parts[2].endswith(".l2:4.ver2"):
post_part += ".l2:4.ver2"
elif modelname_parts[2].endswith("ver2"):
post_part += ".ver2"
elif modelname_parts[2].endswith("ver1"):
post_part += ".ver1"
model_search = "_".join([pre_part, modelname_parts[1], post_part])
if pre_batch is None:
pre_batch = batch
# this is a single-cell fit
if type(cellid) is list:
cellid = cellid[0]
siteid = cellid.split("-")[0]
allsiteids, allcellids = nd.get_batch_sites(batch, modelname_filter=model_search)
allsiteids = [s.split(".")[0] for s in allsiteids]
if (batch==323) and (pre_batch==322):
matchfile=os.path.dirname(__file__) + "/projects/pop_model_scripts/snr_subset_map.csv"
df = pd.read_csv(matchfile, index_col=0)
pre_cellid = df.loc[df.PEG_cellid==cellid, 'A1_cellid'].values[0]
elif (batch==322) and (pre_batch==323):
matchfile=os.path.dirname(__file__) + "/projects/pop_model_scripts/snr_subset_map.csv"
df = pd.read_csv(matchfile, index_col=0)
pre_cellid = df.loc[df.A1_cellid==cellid, 'PEG_cellid'].values[0]
elif siteid in allsiteids:
# don't need to generalize, load from actual fit
pre_cellid = cellid
elif batch in [322, 334]:
pre_cellid = 'ARM029a-07-6'
elif pre_batch == 323:
pre_cellid = 'ARM017a-01-9'
else:
raise ValueError(f"batch {batch} prefit not implemented yet.")
log.info(f"prefit cellid={pre_cellid} prefit batch={pre_batch}")
elif prefit_type == 'site':
# exact same model, just fit for site, now being fit for single cell
pre_parts = modelname_parts[0].split("-")
post_parts = modelname_parts[2].split("-")
model_search = modelname_parts[0] + "%%" + modelname_parts[1] + "%%" + "-".join(post_parts[1:])
pre_cellid = cellid[0]
pre_batch = batch
elif prefit_type is not None:
# this is a single-cell fit
if type(cellid) is list:
cellid = cellid[0]
if prefit_type=='heldout':
if siteid is None:
siteid=cellid.split("-")[0]
cellids, this_perf, alt_cellid, alt_perf = _matching_cells(batch=batch, siteid=siteid)
pre_cellid = [c_alt for c,c_alt in zip(cellids,alt_cellid) if c==cellid][0]
log.info(f"heldout init for {cellid} is {pre_cellid}")
else:
pre_cellid = cellid
log.info(f"matched cellid prefit for {cellid}")
if pre_batch is None:
pre_batch = batch
post_part = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4"
if modelname_parts[2].endswith(".l2:5") or modelname_parts[2].endswith(".l2:5-dstrf"):
post_part += ".l2:5"
elif modelname_parts[2].endswith(".l2:4") or modelname_parts[2].endswith(".l2:4-dstrf"):
post_part += ".l2:4"
elif modelname_parts[2].endswith(".l2:4.ver2"):
post_part += ".l2:4.ver2"
elif modelname_parts[2].endswith("ver2"):
post_part += ".ver2"
modelname_parts[2] = post_part
model_search="_".join(modelname_parts)
elif modelname_parts[1].endswith(".1"):
raise ValueError("deprecated prefit initialization?")
# this is a single-cell fit
if type(cellid) is list:
cellid = cellid[0]
if use_matched:
# determine matched cell for this heldout cell
if siteid is None:
siteid=cellid.split("-")[0]
cellids, this_perf, alt_cellid, alt_perf = _matching_cells(batch=batch, siteid=siteid)
pre_cellid = [c_alt for c,c_alt in zip(cellids,alt_cellid) if c==cellid][0]
log.info(f"matched cell for {cellid} is {pre_cellid}")
else:
pre_cellid = cellid[0]
log.info(f"cellid prefit for {cellid}")
if pre_batch is None:
pre_batch = batch
#postparts = modelname_parts[2].split("-")
#postparts = [s for s in postparts if not(s.startswith("prefit"))]
#modelname_parts[2]="-".join(postparts)
modelname_parts[2] = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4.es20"
model_search="_".join(modelname_parts)
else:
pre_parts = modelname_parts[0].split("-")
post_parts = modelname_parts[2].split("-")
post_part = "tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4.ver2"
model_search = pre_parts[0] + ".pop%%" + modelname_parts[1] + "%%" + post_part
#ozgf.fs100.ch18.pop-loadpop-norm.l1-popev
#wc.18x70.g-fir.1x15x70-relu.70.f-wc.70x80-fir.1x10x80-relu.80.f-wc.80x100-relu.100-wc.100xR-lvl.R-dexp.R
#tfinit.n.lr1e3.et3.rb10.es20-newtf.n.lr1e4.ver2
# hard-coded to use an A1 model!!!!
if pre_batch == 322:
pre_cellid = 'ARM029a-07-6'
elif area == "A1":
pre_cellid = 'ARM029a-07-6'
pre_batch = 322
else:
raise ValueError(f"area {area} prefit not implemented")
log.info(f"model_search: {model_search}")
sql = f"SELECT * FROM Results WHERE batch={pre_batch} and cellid='{pre_cellid}' and modelname like '{model_search}'"
#log.info(sql)
d = nd.pd_query(sql)
#old_uri = adjust_uri_prefix(d['modelpath'][0] + '/modelspec.0000.json')
old_uri = adjust_uri_prefix(d['modelpath'][0])
log.info(f"Importing parameters from {old_uri}")
mspaths = [f"{old_uri}/modelspec.{i:04d}.json" for i in range(modelspec.cell_count)]
print(mspaths)
prefit_ctx = xforms.load_modelspecs([], uris=mspaths, IsReload=False)
#_, prefit_ctx = xform_helper.load_model_xform(
# cellid=pre_cellid, batch=pre_batch,
# modelname=d['modelname'][0], eval_model=False)
new_ctx = load_phi(modelspec, prefit_modelspec=prefit_ctx['modelspec'], copy_layers=copy_layers)
if freeze_early:
new_ctx['freeze_layers'] = list(np.arange(freeze_layer_count))
if prefit_type == 'init':
new_ctx['skip_init'] = True
return new_ctx
# set up axes
plt.figure(figsize=(14, 4))
example_ax = plt.subplot2grid((1, 6), (0, 0), colspan=4)
crd_ax = plt.subplot2grid((1, 6), (0, 4), colspan=1)
drx_ax = plt.subplot2grid((1, 6), (0, 5), colspan=1)
median = True
# ========================================= EXAMPLE SESSION ===========================================================
# define example session
filename = "Cordyceps_2020_02_20"
datestr = "2020-02-20"
window_length = 25
# get relevant parmfiles
sql = "SELECT parmfile, resppath, gDataRaw.id, cellid, gData.svalue, gData.value FROM gDataRaw INNER JOIN gData on (gDataRaw.id=gData.rawid and gData.name='Tar_Frequencies') WHERE runclass=%s and behavior='active' and parmfile like %s"
parmfiles1 = nd.pd_query(sql, ('BVT', filename + "%"))
parmfiles1['date'] = dt.datetime.strptime(datestr, '%Y-%m-%d')
parmfiles1 = parmfiles1.set_index('id')
sql = "SELECT parmfile, resppath, gDataRaw.id, cellid, gData.svalue FROM gDataRaw INNER JOIN gData on (gDataRaw.id=gData.rawid and gData.name='Behave_PumpDuration') WHERE runclass=%s and behavior='active' and parmfile like %s"
parmfiles = nd.pd_query(sql, ('BVT', filename + "%"))
parmfiles['date'] = dt.datetime.strptime(datestr, '%Y-%m-%d')
parmfiles = parmfiles.set_index('id')
parmfiles = parmfiles.rename(columns={'svalue': 'pumpdur'})
# now join on rawid
parmfiles = pd.concat([parmfiles['pumpdur'], parmfiles1], axis=1, join='outer')
parmfiles.loc[parmfiles.svalue.isnull(),
'svalue'] = parmfiles.loc[parmfiles.svalue.isnull(),
'value'].astype(str)
from src.data.load import get_site_ids
from nems.db import pd_query
from warnings import warn
# get CPN sites
all_sites = get_site_ids(316)
region_map = dict()
for site in all_sites.keys():
# pulls the region from celldb
BF_querry = "select area from gCellMaster where siteid=%s"
raw_area = pd_query(BF_querry, params=(site, )).iloc[0, 0]
# Sanitizes region in case of missing values
if raw_area is None:
warn(f'site {site} has undefined region')
print()
continue
else:
area = raw_area.split(',')[0]
if area == '':
print(f'site {site} has undefined region')
continue
elif area not in ('A1', 'PEG'):
print(f'site {site} has unrecognized region:{area}')
continue
region_map[site] = area
def get_model_results_per_state_model(batch=307,
state_list=None,
loader="psth.fs20.pup-ld-",
fitter="_jk.nf20-basic",
basemodel="-ref-psthfr.s_sdexp.S"):
"""
loader = "psth.fs20.pup-ld-"
fitter = "_jk.nf20-basic"
basemodel = "-ref-psthfr.s_sdexp.S"
state_list = ['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
d=get_model_results_per_state_model(batch=307, state_list=state_list,
loader=loader,fitter=fitter,
basemodel=basemodel)
state_list defaults to
['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
"""
if state_list is None:
state_list = [
'st.pup0.beh0', 'st.pup0.beh', 'st.pup.beh0', 'st.pup.beh'
]
modelnames = [loader + s + basemodel + fitter for s in state_list]
celldata = nd.get_batch_cells(batch=batch)
cellids = celldata['cellid'].tolist()
isolation = [
nd.get_isolation(cellid=c, batch=batch).loc[0, 'min_isolation']
for c in cellids
]
if state_list[-1].endswith('fil') or state_list[-1].endswith('pas'):
include_AP = True
else:
include_AP = False
d = pd.DataFrame(columns=[
'cellid', 'modelname', 'state_sig', 'state_chan', 'MI', 'isolation',
'r', 'r_se', 'd', 'g', 'sp', 'state_chan_alt'
])
new_sdexp = False
for mod_i, m in enumerate(modelnames):
print('Loading modelname: ', m)
modelspecs = nems_db.params._get_modelspecs(cellids,
batch,
m,
multi='mean')
for modelspec in modelspecs:
meta = ms.get_modelspec_metadata(modelspec)
phi = list(modelspec[0]['phi'].keys())
c = meta['cellid']
iso = isolation[cellids.index(c)]
state_mod = meta['state_mod']
state_mod_se = meta['se_state_mod']
state_chans = meta['state_chans']
if 'g' in phi:
dc = modelspec[0]['phi']['d']
gain = modelspec[0]['phi']['g']
elif ('amplitude_g' in phi) & ('amplitude_d' in phi):
new_sdexp = True
dc = None
gain = None
g_amplitude = modelspec[0]['phi']['amplitude_g']
g_base = modelspec[0]['phi']['base_g']
g_kappa = modelspec[0]['phi']['kappa_g']
g_offset = modelspec[0]['phi']['offset_g']
d_amplitude = modelspec[0]['phi']['amplitude_d']
d_base = modelspec[0]['phi']['base_d']
d_kappa = modelspec[0]['phi']['kappa_d']
d_offset = modelspec[0]['phi']['offset_d']
gain_mod = None
dc_mod = None
if 'state_mod_gain' in meta.keys():
gain_mod = meta['state_mod_gain']
dc_mod = meta['state_mod_dc']
if dc is not None:
sp = modelspec[0]['phi'].get('sp', np.zeros(gain.shape))
if dc.ndim > 1:
dc = dc[0, :]
gain = gain[0, :]
sp = sp[0, :]
a_count = 0
p_count = 0
for j, sc in enumerate(state_chans):
if gain is not None:
gain_val = gain[j]
dc_val = dc[j]
sp_val = sp[j]
else:
gain_val = None
dc_val = None
sp_val = None
r = {
'cellid': c,
'state_chan': sc,
'modelname': m,
'isolation': iso,
'state_sig': state_list[mod_i],
'g': gain_val,
'd': dc_val,
'sp': sp_val,
'MI': state_mod[j],
'r': meta['r_test'][0],
'r_se': meta['se_test'][0]
}
if new_sdexp:
r.update({
'g_amplitude': g_amplitude[0, j],
'g_base': g_base[0, j],
'g_kappa': g_kappa[0, j],
'g_offset': g_offset[0, j],
'd_amplitude': d_amplitude[0, j],
'd_base': d_base[0, j],
'd_kappa': d_kappa[0, j],
'd_offset': d_offset[0, j]
})
if gain_mod is not None:
r.update({'gain_mod': gain_mod[j], 'dc_mod': dc_mod[j]})
d = d.append(r, ignore_index=True)
l = len(d) - 1
if include_AP and sc.startswith("FILE_"):
siteid = c.split("-")[0]
fn = "%" + sc.replace("FILE_", "") + "%"
sql = "SELECT * FROM gDataRaw WHERE cellid=%s" +\
" AND parmfile like %s"
dcellfile = nd.pd_query(sql, (siteid, fn))
if dcellfile.loc[0]['behavior'] == 'active':
a_count += 1
d.loc[l,
'state_chan_alt'] = "ACTIVE_{}".format(a_count)
else:
p_count += 1
d.loc[l,
'state_chan_alt'] = "PASSIVE_{}".format(p_count)
else:
d.loc[l, 'state_chan_alt'] = d.loc[l, 'state_chan']
#d['r_unique'] = d['r'] - d['r0']
#d['MI_unique'] = d['MI'] - d['MI0']
return d
def penetration_map(sites,
equal_aspect=False,
flip_X=False,
flatten=False,
flip_YZ=False,
landmarks=None):
"""
Plots a 3d map of the list of specified sites, displaying the best frequency as color, and the brain region as
maker type (NA: circle, A1: triangle, PEG: square).
The site location, brain area and best frequency are extracted from celldb, specifically from the penetration (for
coordinates and rotations) and the penetration (for area and best frequency) sites. If no coordinates are found the
site is ignored. no BF is displayed as an empty marker.
The values in the plot increase in direction Posterior -> Anterior, Medial -> Lateral and Ventral -> Dorsal. The
antero posterior (X) axis can be flipped with the according parameter.
:param sites: list of str specifying sites, with the format "ABC001a"
:param equal_aspect: boolean. whether to constrain the data to a cubic/square space i.e. equal dimensions in XYZ/XY
:flip_X: Boolean. Flips the direction labels for the antero-posterior (X) axis. The default is A > P .
Y. Lateral > Medial, Z. Dorsal > Ventral.
:flatten: Boolean. PCA 2d projection. Work in progress.
:flip_YZ: Boolean. Flips the direction and labels of the YZ principal component when flattening.
:landmarks: dict of vectors, where the key specifies the landmark name, and the vector has the values
[x0, y0, z0, x, y, z, tilt, rot]. If the landmark name is 'OccCrest' or 'MidLine' uses the AP and ML values as zeros
respectively.
:return: matplotlib figure
"""
area_marker = {'NA': 'o', 'A1': '^', 'PEG': 's'}
coordinates = list()
best_frequencies = list()
areas = list()
good_sites = list()
# get values from cell db and transforms into coordinates
for pp, site in enumerate(sites):
# gets the penetrations MT coordinates
penetration = site[0:6]
coord_querry = "select ecoordinates from gPenetration where penname=%s"
raw_coords = db.pd_query(coord_querry, params=(penetration, )).iloc[0,
0]
all_coords = np.asarray(raw_coords.split(',')).astype(float).squeeze()
# Dimensions X:Antero-Posterior Y:Medio-Lateral Z:Dorso-Ventral
MT_0 = all_coords[0:3]
MT = all_coords[3:6]
tilt = radians(all_coords[6])
rotation = radians(all_coords[7])
# rejects sites with no coordinates
no_ref = np.all(MT_0 == 0)
no_val = np.all(MT == 0)
if no_ref or no_val:
print(
f'skipping penetration {penetration}, No coordinates specified'
)
continue
# defines tilt and rotation matrices, tilt around X axis, rotation around Z axis
tilt_mat = np.asarray([[1, 0, 0], [0, cos(tilt),
sin(tilt)],
[0, -sin(tilt), cos(tilt)]])
rot_mat = np.asarray([[cos(rotation), sin(rotation), 0],
[-sin(rotation),
cos(rotation), 0], [0, 0, 1]])
# calculates the relative MT coordinates and rotates.
MT_rel_rot = rot_mat @ tilt_mat @ (MT - MT_0)
# get the first value of BF from cellDB TODO this is a temporary cludge, in the future, with all BFs set, a more elegant approach is required
BF_querry = "select bf, area from gCellMaster where siteid=%s"
try:
raw_BF, raw_area = db.pd_query(BF_querry,
params=(site, )).iloc[0, :]
except:
raw_BF = None
raw_area = None
# Sanitize best frequency in case of missing values
if raw_BF is None:
print(f'site {site} has undefined best frequency')
BF = 0
else:
BF = int(raw_BF.split(',')[0])
if BF == 0:
print(f'site {site} has undefined best frequency')
# Sanitizes region in case of missing values
if raw_area is None:
print(f'site {site} has undefined region')
area = 'NA'
else:
area = raw_area.split(',')[0]
if area == '':
print(f'site {site} has undefined region')
area = 'NA'
elif area not in ('A1', 'PEG'):
print(f'site {site} has unrecognized region: {area}')
area = 'NA'
coordinates.append(MT_rel_rot)
best_frequencies.append(BF)
areas.append(area)
good_sites.append(site)
# adds manual landmarks specified in dictionary
if landmarks is not None:
X0 = []
Y0 = []
for landname, all_coords in landmarks.items():
all_coords = np.asarray(all_coords)
MT_0 = all_coords[0:3]
MT = all_coords[3:6]
tilt = radians(all_coords[6])
rotation = radians(all_coords[7])
# defines tilt and rotation matrices, tilt around X axis, rotation around Z axis
tilt_mat = np.asarray([[1, 0, 0], [0, cos(tilt),
sin(tilt)],
[0, -sin(tilt), cos(tilt)]])
rot_mat = np.asarray([[cos(rotation),
sin(rotation), 0],
[-sin(rotation),
cos(rotation), 0], [0, 0, 1]])
# calculates the relative MT coordinates and rotates.
MT_rel_rot = rot_mat @ tilt_mat @ (MT - MT_0)
coordinates.append(MT_rel_rot)
best_frequencies.append(0)
areas.append('NA')
# pads with spaces and holds 3 letter, for consistent naming with sites
good_sites.append(f' {landname[0:3]}')
#saves values as zero reference if correct landname
if landname == 'OccCrest': X0.append(MT_rel_rot[0])
if landname == 'MidLine': Y0.append(MT_rel_rot[1])
coordinates = np.stack(coordinates, axis=1)
best_frequencies = np.asarray(best_frequencies)
areas = np.asarray(areas)
good_sites = np.asarray(good_sites)
# centers data and transforms cm to mm
center = np.mean(coordinates, axis=1)
# uses landmarks as zero values if any
if landmarks is not None:
if X0: center[0] = X0[0]
if Y0: center[1] = Y0[0]
coordinates = coordinates - center[:, None]
coordinates = coordinates * 10
# defines BF colormap range if valid best frequencies are available.
vmax = best_frequencies.max() if best_frequencies.max() > 0 else 32000
vmin = best_frequencies[
best_frequencies != 0].min() if best_frequencies.min() > 0 else 100
if flatten is False:
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
if equal_aspect:
X, Y, Z = coordinates
# Create cubic bounding box to simulate equal aspect ratio
max_range = np.array(
[X.max() - X.min(),
Y.max() - Y.min(),
Z.max() - Z.min()]).max()
Xb = 0.5 * max_range * np.mgrid[-1:2:2, -1:2:2, -1:2:2][0].flatten(
) + 0.5 * (X.max() + X.min())
Yb = 0.5 * max_range * np.mgrid[-1:2:2, -1:2:2, -1:2:2][1].flatten(
) + 0.5 * (Y.max() + Y.min())
Zb = 0.5 * max_range * np.mgrid[-1:2:2, -1:2:2, -1:2:2][2].flatten(
) + 0.5 * (Z.max() + Z.min())
for xb, yb, zb in zip(Xb, Yb, Zb):
ax.plot([xb], [yb], [zb], 'w')
for area in set(areas):
coord_subset = coordinates[:, areas == area]
BF_subset = best_frequencies[areas == area]
site_subset = good_sites[areas == area]
X, Y, Z = coord_subset
p = ax.scatter(X,
Y,
Z,
s=100,
marker=area_marker[area],
edgecolor='black',
c=BF_subset,
cmap='inferno',
norm=colors.LogNorm(vmin=vmin, vmax=vmax))
for coord, site in zip(coord_subset.T, site_subset):
x, y, z = coord
ax.text(x, y, z, site[3:6])
# formats axis
ax.set_xlabel('anterior posterior (mm)')
ax.set_ylabel('Medial Lateral (mm)')
ax.set_zlabel('Dorsal ventral (mm)')
fig.canvas.draw()
x_tick_loc = ax.get_xticks().tolist()
xlab = [f'{x:.1f}' for x in x_tick_loc]
if flip_X:
xlab[0] = 'A'
xlab[-1] = 'P'
else:
xlab[0] = 'P'
xlab[-1] = 'A'
_ = ax.xaxis.set_major_locator(mticker.FixedLocator(x_tick_loc))
_ = ax.set_xticklabels(xlab)
y_tick_loc = ax.get_yticks().tolist()
ylab = [f'{x:.1f}' for x in y_tick_loc]
ylab[0] = 'M'
ylab[-1] = 'L'
_ = ax.yaxis.set_major_locator(mticker.FixedLocator(y_tick_loc))
_ = ax.set_yticklabels(ylab)
z_tick_loc = ax.get_zticks().tolist()
zlab = [f'{x:.1f}' for x in z_tick_loc]
zlab[0] = 'V'
zlab[-1] = 'D'
_ = ax.zaxis.set_major_locator(mticker.FixedLocator(z_tick_loc))
_ = ax.set_zticklabels(zlab)
elif flatten is True:
# flattens doing a PCA over the Y and Z dimensions, i.e. medio-lateral and dorso-ventral.
# this keeps the anteroposterior orientations to help locate the flattened projection
pc1 = PCA().fit_transform(coordinates[1:, :].T)[:, 0]
# uses midline zero as pc1 zero
if landmarks is not None and 'MidLine' in landmarks.keys():
pc1 = pc1 - pc1[np.argwhere(coordinates[1, :] == 0)].squeeze()
flat_coords = np.stack((coordinates[0, :], pc1), axis=0)
# FLips data on axes since 2d plots cannot be rotated interactively
fx = -1 if flip_X is True else 1
fyz = -1 if flip_YZ is True else 1
flat_coords = flat_coords * np.array([[fx], [fyz]])
fig, ax = plt.subplots()
for area in set(areas):
flat_coords_subset = flat_coords[:, areas == area]
BF_subset = best_frequencies[areas == area]
site_subset = good_sites[areas == area]
X, Y = flat_coords_subset
p = ax.scatter(X,
Y,
s=100,
marker=area_marker[area],
edgecolor='black',
c=BF_subset,
cmap='inferno',
norm=colors.LogNorm(vmin=vmin, vmax=vmax))
for coord, site in zip(flat_coords_subset.T, site_subset):
x, y = coord
ax.text(x, y, site[3:6])
# formats axis
if equal_aspect:
ax.axis('equal')
ax.set_xlabel('anterior posterior (mm)')
ax.set_ylabel('1PC_YZ (mm)')
cbar = fig.colorbar(p)
cbar.ax.set_ylabel('BF (Hz)', rotation=-90, va="top")
return fig, coordinates
def plot_collapsed_ref_tar(animal, site, cellids=None):
site += "%"
sql = "SELECT DISTINCT cellid, rawid, respfile FROM sCellFile WHERE cellid like %s AND runclassid=%s"
d = nd.pd_query(sql, params=(
site,
42,
))
mfile = []
for f in np.unique(d.respfile):
f_ = f.split('.')[0]
mfile.append('/auto/data/daq/{0}/{1}/{2}'.format(
animal, site[:-2], f_))
if cellids is None:
cellid = np.unique(d.cellid).tolist()
else:
cellid = cellids
options = {
"siteid": site[:-1],
'cellid': cellids,
"mfilename": mfile,
'stim': False,
'pupil': True,
'rasterfs': 1000
}
uri = nb.baphy_load_recording_uri(**options)
rec = Recording.load(uri)
all_pupil = rec['pupil']._data
ncols = len(mfile)
if cellids is None:
cellids = rec['resp'].chans
for c in cellids:
f, ax = plt.subplots(1, ncols, sharey=True, figsize=(12, 5))
ref_base = 0
for i, mf in enumerate(mfile):
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
R = rec.copy()
R['resp'] = R['resp'].rasterize(fs=20)
R['resp'].fs = 20
R = R.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
R = R.and_mask(['HIT_TRIAL']).apply_mask(reset_epochs=True)
resp = R['resp'].extract_channels([c])
tar_reps = resp.extract_epoch('TARGET').shape[0]
tar_m = np.nanmean(resp.extract_epoch('TARGET'),
0).squeeze() * R['resp'].fs
tar_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('TARGET'),
0).squeeze() / np.sqrt(tar_reps)
ref_reps = resp.extract_epoch('REFERENCE').shape[0]
ref_m = np.nanmean(resp.extract_epoch('REFERENCE'),
0).squeeze() * R['resp'].fs
ref_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('REFERENCE'),
0).squeeze() / np.sqrt(ref_reps)
# plot psth's
time = np.linspace(0, len(tar_m) / R['resp'].fs, len(tar_m))
ax[i].plot(time, tar_m, color='red', lw=2)
ax[i].fill_between(time,
tar_m + tar_sem,
tar_m - tar_sem,
color='coral')
time = np.linspace(0, len(ref_m) / R['resp'].fs, len(ref_m))
ax[i].plot(time, ref_m, color='blue', lw=2)
ax[i].fill_between(time,
ref_m + ref_sem,
ref_m - ref_sem,
color='lightblue')
# set title
ax[i].set_title(fn, fontsize=8)
# set labels
ax[i].set_xlabel('Time (s)')
ax[i].set_ylabel('Spk / sec')
# get raster plot baseline
base = np.max(np.concatenate((tar_m + tar_sem, ref_m + ref_sem)))
if base > ref_base:
ref_base = base
for i, mf in enumerate(mfile):
# plot the rasters
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
rast_rec = rec.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
rast_rec = rast_rec.and_mask(['HIT_TRIAL'
]).apply_mask(reset_epochs=True)
rast = rast_rec['resp'].extract_channels([c])
ref_times = np.where(rast.extract_epoch('REFERENCE').squeeze())
base = ref_base
ref_pupil = np.nanmean(
rast_rec['pupil'].extract_epoch('REFERENCE'), -1)
xoffset = rast.extract_epoch(
'TARGET').shape[-1] / rec['resp'].fs + 0.01
ax[i].plot(ref_pupil / np.max(all_pupil) + xoffset,
np.linspace(base, int(base * 2), len(ref_pupil)),
color='k')
ax[i].axvline(xoffset + np.median(all_pupil / np.max(all_pupil)),
linestyle='--',
color='lightgray')
#import pdb; pdb.set_trace()
if ref_times[0].size != 0:
max_rep = ref_pupil.shape[0] - 1
ref_locs = ref_times[0] * (base / max_rep)
ref_locs = ref_locs + base
ax[i].plot(ref_times[1] / rec['resp'].fs,
ref_locs,
'|',
color='blue',
markersize=1)
tar_times = np.where(rast.extract_epoch('TARGET').squeeze())
tar_pupil = np.nanmean(rast_rec['pupil'].extract_epoch('TARGET'),
-1)
tar_base = np.max(ref_locs) + 1
ax[i].plot(tar_pupil / np.max(all_pupil) + xoffset,
np.linspace(tar_base, tar_base + base, len(tar_pupil)),
color='k')
if tar_times[0].size != 0:
max_rep = tar_pupil.shape[0] - 1
tar_locs = tar_times[0] * (base / max_rep)
tar_locs = tar_locs + tar_base
ax[i].plot(tar_times[1] / rec['resp'].fs,
tar_locs,
'|',
color='red',
markersize=1)
# set ylim
#ax[i].set_ylim((0, top))
# set plot aspect
#asp = np.diff(ax[i].get_xlim())[0] / np.diff(ax[i].get_ylim())[0]
#ax[i].set_aspect(asp / 2)
f.suptitle(c, fontsize=8)
f.tight_layout()
120201, 120207, 120209, 120211, 120214, 120234, 120234, 120254, 120256,
120258, 120260, 120272, 120273, 120274, 120275, 120293, 120283, 120285,
120286, 120289, 120290, 120293, 120310, 120311, 120312, 120313, 120314,
120316, 120317, 120435, 120436, 120437
]
# ================================= batch 307 ====================================
perfile_df = pd.read_csv(os.path.join(fpath, str(307), 'd_pup_fil_sdexp.csv'),
index_col=0)
df_307 = pd.DataFrame()
cells_307 = nd.get_batch_cells(307).cellid
for cellid in cells_307:
_, rawid = nd.get_stable_batch_cells(batch=307, cellid=cellid)
sql = "SELECT value, svalue, rawid from gData where name='Trial_TargetIdxFreq' and rawid in {}".format(
tuple(rawid))
d = nd.pd_query(sql, params=())
sql = "SELECT value, svalue, rawid from gData where name='Trial_RelativeTarRefdB' and rawid in {}".format(
tuple(rawid))
d2 = nd.pd_query(sql, params=())
sql = "SELECT behavior, id from gDataRaw where id in {0}".format(
tuple(rawid))
da = nd.pd_query(sql)
d = d[d.rawid.isin(
[r for r in da.id if da[da.id == r]['behavior'].values == 'active'])]
d2 = d2[d2.rawid.isin(
[r for r in da.id if da[da.id == r]['behavior'].values == 'active'])]
d2.columns = [c + '_rel' for c in d2.columns]
d = pd.concat([d, d2], axis=1)
pf_labels = np.unique([
#cellid="BRT038b-30-1"
#modelname="ozgf.fs100.ch18-ld-sev_dlog-wc.18x3-fir.1x15x3-relu.3-wc.3x1-lvl.1-dexp.1_tf.n.rb5"
#modelname="ozgf.fs100.ch18-ld-sev_dlog-wc.18x16.g-fir.4x15x4-relu.4-wc.4x1-lvl.1-dexp.1_tf.n.rb5"
#modelname='ozgf.fs100.ch18-ld-sev_dlog-wc.18x12-fir.4x15x3-relu.3-wc.3x1-lvl.1-dexp.1_tf.n.rb5'
cellid="BRT033b-12-3"
batch=308
modelname="ozgf.fs100.ch18-ld-sev_dlog-wc.18x16-fir.4x15x4-relu.4-wc.4x1-lvl.1-dexp.1_tf.n.rb5"
# modelname="ozgf.fs100.ch18-ld-sev_dlog-wc.18x2.g-fir.2x15-relu.1-lvl.1-dexp.1_tf.n.rb5"
xfspec, ctx = xhelp.load_model_xform(cellid, batch, modelname)
ex = gui.browse_xform_fit(ctx, xfspec)
########################################################
########################################################
#########################################################
########################################################
import nems.db as nd
batch=308
performance_data = nd.pd_query("SELECT modelname,cellid,r_test,r_floor FROM Results WHERE batch={}".format(batch))
performance_data['significant']=performance_data['r_test'] > 2*performance_data['r_floor']
performance_data['zero_test']=(performance_data['r_test']==0)
performance_data.groupby(['modelname'])['r_test','significant','zero_test'].mean()
performance_data.groupby(['modelname'])['zero_test'].mean()
"""
Summary plots of r_test, gain vs. DC, and MI.
"""
import nems.db as nd
import matplotlib.pyplot as plt
import pandas as pd
batches = [289, 294, 323]
modelnames = ['ns.fs4.pup-ld-st.pup-hrc-psthfr_sdexp.SxR.bound_jk.nf10-basic', 'ns.fs4.pup-ld-st.pup0-hrc-psthfr_sdexp.SxR.bound_jk.nf10-basic',
'ns.fs4.pup.voc-ld-st.pup-hrc-psthfr_sdexp.SxR.bound_jk.nf10-basic', 'ns.fs4.pup.voc-ld-st.pup0-hrc-psthfr_sdexp.SxR.bound_jk.nf10-basic']
sql = "SELECT r_test, se_test, cellid, modelname, batch from Results WHERE modelname in {0} and batch in {1}".format(tuple(modelnames), tuple(batches))
results = nd.pd_query(sql)
results['state_mod'] = ['st.pup' if 'st.pup0' not in s else 'st.pup0' for s in results['modelname']]
results = results[results.batch==323]
r = results.pivot(columns='state_mod', index='cellid')
rdiff = r.loc[:, pd.IndexSlice['r_test', 'st.pup']] - r.loc[:, pd.IndexSlice['r_test', 'st.pup0']]
se = r.loc[:, pd.IndexSlice['se_test', 'st.pup']] + r.loc[:, pd.IndexSlice['se_test', 'st.pup0']]
sig_mask = rdiff > se
f, ax = plt.subplots(1, 1, figsize=(6, 6))
ax.scatter(r.loc[:, pd.IndexSlice['r_test', 'st.pup0']] ** 2,
r.loc[:, pd.IndexSlice['r_test', 'st.pup']] ** 2,
marker='o', edgecolor='white', color='grey')
ax.scatter(r.loc[sig_mask, pd.IndexSlice['r_test', 'st.pup0']] ** 2,
r.loc[sig_mask, pd.IndexSlice['r_test', 'st.pup']] ** 2,
import numpy as np
from scipy.signal import convolve2d, butter, sosfilt
from nems import db
from nems.utils import smooth
from nems_lbhb.xform_wrappers import generate_recording_uri
from nems_lbhb.baphy_experiment import BAPHYExperiment
from nems_lbhb.baphy_io import load_continuous_openephys
from nems_lbhb.plots import plot_waveforms_64D
USE_DB = False
if USE_DB:
expt_name = "TNC020a11_p_BNB"
expt_name = "TNC017a10_p_BNB"
dparm = db.pd_query(
f"SELECT * FROM gDataRaw where parmfile like '{expt_name}%'")
parmfile = dparm.resppath[0] + dparm.parmfile[0]
else:
#hard-code path to parmfile
#parmfile = "/auto/data/daq/Teonancatl/TNC018/TNC018a16_p_BNB.m"
#parmfile = "/auto/data/daq/Teonancatl/TNC020/TNC020a11_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC017/TNC017a03_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC017/TNC017a10_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC016/TNC016a03_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC018/TNC018a03_p_BNB.m"
parmfile = "/auto/data/daq/Tartufo/TAR010/TAR010a03_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC006/TNC006a03_p_BNB.m"
parmfile = "/auto/data/daq/Teonancatl/TNC006/TNC006a19_p_BNB.m"
## load the recording
parmfile = "/auto/data/daq/Tartufo/TAR010/TAR010a03_p_BNB.m"
def calc_psth_metrics(batch, cellid, parmfile=None, paths=None):
start_win_offset = 0 # Time (in sec) to offset the start of the window used to calculate threshold, exitatory percentage, and inhibitory percentage
if parmfile:
manager = BAPHYExperiment(parmfile)
else:
manager = BAPHYExperiment(cellid=cellid, batch=batch)
options = ohel.get_load_options(
batch) #gets options that will include gtgram if batch=339
rec = manager.get_recording(**options)
area_df = db.pd_query(
f"SELECT DISTINCT area FROM sCellFile where cellid like '{manager.siteid}%%'"
)
area = area_df.area.iloc[0]
if rec['resp'].epochs[rec['resp'].epochs['name'] ==
'PASSIVE_EXPERIMENT'].shape[0] >= 2:
rec = ohel.remove_olp_test(rec)
rec['resp'] = rec['resp'].extract_channels([cellid])
resp = copy.copy(rec['resp'].rasterize())
rec['resp'].fs = 100
norm_spont, SR, STD = ohel.remove_spont_rate_std(resp)
params = ohel.get_expt_params(resp, manager, cellid)
epcs = rec['resp'].epochs[rec['resp'].epochs['name'] ==
'PreStimSilence'].copy()
ep2 = rec['resp'].epochs[rec['resp'].epochs['name'] ==
'PostStimSilence'].iloc[0].copy()
params['prestim'], params['poststim'] = epcs.iloc[0][
'end'], ep2['end'] - ep2['start']
params['lenstim'] = ep2['end']
stim_epochs = ep.epoch_names_matching(resp.epochs, 'STIM_')
if paths and cellid[:3] == 'TBR':
print(f"Deprecated, run on {cellid} though...")
stim_epochs, rec, resp = ohel.path_tabor_get_epochs(
stim_epochs, rec, resp, params)
epoch_repetitions = [resp.count_epoch(cc) for cc in stim_epochs]
full_resp = np.empty((max(epoch_repetitions), len(stim_epochs),
(int(params['lenstim']) * rec['resp'].fs)))
full_resp[:] = np.nan
for cnt, epo in enumerate(stim_epochs):
resps_list = resp.extract_epoch(epo)
full_resp[:resps_list.shape[0], cnt, :] = resps_list[:, 0, :]
#Calculate a few metrics
corcoef = ohel.calc_base_reliability(full_resp)
avg_resp = ohel.calc_average_response(full_resp, params)
snr = compute_snr(resp)
#Grab and label epochs that have two sounds in them (no null)
presil, postsil = int(params['prestim'] * rec['resp'].fs), int(
params['poststim'] * rec['resp'].fs)
twostims = resp.epochs[resp.epochs['name'].str.count('-0-1') == 2].copy()
ep_twostim = twostims.name.unique().tolist()
ep_twostim.sort()
ep_names = resp.epochs[resp.epochs['name'].str.contains('STIM_')].copy()
ep_names = ep_names.name.unique().tolist()
ep_types = list(map(ohel.label_ep_type, ep_names))
ep_df = pd.DataFrame({'name': ep_names, 'type': ep_types})
cell_df = []
for cnt, stimmy in enumerate(ep_twostim):
kind = ohel.label_pair_type(stimmy)
seps = (stimmy.split('_')[1], stimmy.split('_')[2])
BG, FG = seps[0].split('-')[0][2:], seps[1].split('-')[0][2:]
Aepo, Bepo = 'STIM_' + seps[0] + '_null', 'STIM_null_' + seps[1]
rAB = resp.extract_epoch(stimmy)
rA, rB = resp.extract_epoch(Aepo), resp.extract_epoch(Bepo)
fn = lambda x: np.atleast_2d(sp.smooth(x.squeeze(), 3, 2) - SR)
rAsm = np.squeeze(np.apply_along_axis(fn, 2, rA))
rBsm = np.squeeze(np.apply_along_axis(fn, 2, rB))
rABsm = np.squeeze(np.apply_along_axis(fn, 2, rAB))
rA_st, rB_st = rAsm[:, presil:-postsil], rBsm[:, presil:-postsil]
rAB_st = rABsm[:, presil:-postsil]
rAm, rBm = np.nanmean(rAsm, axis=0), np.nanmean(rBsm, axis=0)
rABm = np.nanmean(rABsm, axis=0)
AcorAB = np.corrcoef(
rAm, rABm)[0, 1] # Corr between resp to A and resp to dual
BcorAB = np.corrcoef(
rBm, rABm)[0, 1] # Corr between resp to B and resp to dual
A_FR, B_FR, AB_FR = np.nanmean(rA_st), np.nanmean(rB_st), np.nanmean(
rAB_st)
min_rep = np.min(
(rA.shape[0],
rB.shape[0])) #only will do something if SoundRepeats==Yes
lin_resp = np.nanmean(rAsm[:min_rep, :] + rBsm[:min_rep, :], axis=0)
supp = np.nanmean(lin_resp - AB_FR)
AcorLin = np.corrcoef(
rAm, lin_resp)[0, 1] # Corr between resp to A and resp to lin
BcorLin = np.corrcoef(
rBm, lin_resp)[0, 1] # Corr between resp to B and resp to lin
Apref, Bpref = AcorAB - AcorLin, BcorAB - BcorLin
pref = Apref - Bpref
# if params['Binaural'] == 'Yes':
# dA, dB = ohel.get_binaural_adjacent_epochs(stimmy)
#
# rdA, rdB = resp.extract_epoch(dA), resp.extract_epoch(dB)
# rdAm = np.nanmean(np.squeeze(np.apply_along_axis(fn, 2, rdA))[:, presil:-postsil], axis=0)
# rdBm = np.nanmean(np.squeeze(np.apply_along_axis(fn, 2, rdB))[:, presil:-postsil], axis=0)
#
# ABcordA = np.corrcoef(rABm, rdAm)[0, 1] # Corr between resp to AB and resp to BG swap
# ABcordB = np.corrcoef(rABm, rdBm)[0, 1] # Corr between resp to AB and resp to FG swap
cell_df.append({
'epoch': stimmy,
'kind': kind,
'BG': BG,
'FG': FG,
'AcorAB': AcorAB,
'BcorAB': BcorAB,
'AcorLin': AcorLin,
'BcorLin': BcorLin,
'Apref': Apref,
'Bpref': Bpref,
'pref': pref,
'combo_FR': AB_FR,
'bg_FR': A_FR,
'fg_FR': B_FR,
'supp': supp
})
cell_df = pd.DataFrame(cell_df)
cell_df['SR'], cell_df['STD'] = SR, STD
# cell_df['corcoef'], cell_df['avg_resp'], cell_df['snr'] = corcoef, avg_resp, snr
cell_df.insert(loc=0, column='area', value=area)
return cell_df
# COMPUTE ALL FOLLOWING metrics using smoothed driven rate
# est, val = rec.split_using_epoch_occurrence_counts(rec,epoch_regex='^STIM_')
val = rec.copy()
val['resp'] = val['resp'].rasterize()
val['stim'] = val['stim'].rasterize()
val = preproc.average_away_epoch_occurrences(val, epoch_regex='^STIM_')
# smooth and subtract SR
fn = lambda x: np.atleast_2d(sp.smooth(x.squeeze(), 3, 2) - SR)
val['resp'] = val['resp'].transform(fn)
val['resp'] = ohel.add_stimtype_epochs(val['resp'])
if val['resp'].count_epoch('REFERENCE'):
epochname = 'REFERENCE'
else:
epochname = 'TRIAL'
sts = val['resp'].epochs['start'].copy()
nds = val['resp'].epochs['end'].copy()
sts_rec = rec['resp'].epochs['start'].copy()
val['resp'].epochs['end'] = val['resp'].epochs['start'] + params['prestim']
ps = val['resp'].select_epochs([epochname]).as_continuous()
ff = np.isfinite(ps)
SR_av = ps[ff].mean() * resp.fs
SR_av_std = ps[ff].std() * resp.fs
# Compute max over single-voice trials
val['resp'].epochs['end'] = nds
val['resp'].epochs['start'] = sts
val['resp'].epochs[
'start'] = val['resp'].epochs['start'] + params['prestim']
TotalMax = np.nanmax(val['resp'].as_continuous())
ps = np.hstack((val['resp'].extract_epoch('10').flatten(),
val['resp'].extract_epoch('01').flatten()))
SinglesMax = np.nanmax(ps)
# Compute threshold, exitatory percentage, and inhibitory percentage
prestim, poststim = params['prestim'], params['poststim']
val['resp'].epochs['end'] = nds
val['resp'].epochs['start'] = sts
val['resp'].epochs[
'start'] = val['resp'].epochs['start'] + prestim + start_win_offset
val['resp'].epochs['end'] = val['resp'].epochs['end'] - poststim
thresh = np.array(((SR + SR_av_std) / resp.fs, (SR - SR_av_std) / resp.fs))
thresh = np.array((SR / resp.fs + 0.1 * (SinglesMax - SR / resp.fs),
(SR - SR_av_std) / resp.fs))
# SR/resp.fs - 0.5 * (np.nanmax(val['resp'].as_continuous()) - SR/resp.fs)]
types = ['10', '01', '20', '02', '11', '12', '21', '22']
excitatory_percentage = {}
inhibitory_percentage = {}
Max = {}
Mean = {}
for _type in types:
if _type in val['resp'].epochs.name.values:
ps = val['resp'].extract_epoch(_type).flatten()
ff = np.isfinite(ps)
excitatory_percentage[_type] = (ps[ff] >
thresh[0]).sum() / ff.sum()
inhibitory_percentage[_type] = (ps[ff] <
thresh[1]).sum() / ff.sum()
Max[_type] = ps[ff].max() / SinglesMax
Mean[_type] = ps[ff].mean()
# Compute threshold, exitatory percentage, and inhibitory percentage just over onset time
# restore times
val['resp'].epochs['end'] = nds
val['resp'].epochs['start'] = sts
# Change epochs to stimulus onset times
val['resp'].epochs['start'] = val['resp'].epochs['start'] + prestim
val['resp'].epochs['end'] = val['resp'].epochs['start'] + prestim + .5
excitatory_percentage_onset = {}
inhibitory_percentage_onset = {}
Max_onset = {}
for _type in types:
ps = val['resp'].extract_epoch(_type).flatten()
ff = np.isfinite(ps)
excitatory_percentage_onset[_type] = (ps[ff] >
thresh[0]).sum() / ff.sum()
inhibitory_percentage_onset[_type] = (ps[ff] <
thresh[1]).sum() / ff.sum()
Max_onset[_type] = ps[ff].max() / SinglesMax
# find correlations between double and single-voice responses
val['resp'].epochs['end'] = nds
val['resp'].epochs['start'] = sts
val['resp'].epochs['start'] = val['resp'].epochs['start'] + prestim
rec['resp'].epochs['start'] = rec['resp'].epochs['start'] + prestim
# over stim on time to end + 0.5
val['linmodel'] = val['resp'].copy()
val['linmodel']._data = np.full(val['linmodel']._data.shape, np.nan)
types = ['11', '12', '21', '22']
epcs = val['resp'].epochs[val['resp'].epochs['name'].str.contains(
'STIM')].copy()
epcs['type'] = epcs['name'].apply(ohel.label_ep_type)
names = [[n.split('_')[1], n.split('_')[2]] for n in epcs['name']]
EA = np.array([n[0] for n in names])
EB = np.array([n[1] for n in names])
r_dual_B, r_dual_A, r_dual_B_nc, r_dual_A_nc = {}, {}, {}, {}
r_dual_B_bal, r_dual_A_bal = {}, {}
r_lin_B, r_lin_A, r_lin_B_nc, r_lin_A_nc = {}, {}, {}, {}
r_lin_B_bal, r_lin_A_bal = {}, {}
N_ac = 200
full_resp = rec['resp'].rasterize()
full_resp = full_resp.transform(fn)
for _type in types:
inds = np.nonzero(epcs['type'].values == _type)[0]
rA_st, rB_st, r_st, rA_rB_st = [], [], [], []
init = True
for ind in inds:
# for each dual-voice response
r = val['resp'].extract_epoch(epcs.iloc[ind]['name'])
if np.any(np.isfinite(r)):
print(epcs.iloc[ind]['name'])
# Find the indicies of single-voice responses that match this dual-voice response
indA = np.where((EA[ind] == EA) & (EB == 'null'))[0]
indB = np.where((EB[ind] == EB) & (EA == 'null'))[0]
if (len(indA) > 0) & (len(indB) > 0):
# from pdb import set_trace
# set_trace()
rA = val['resp'].extract_epoch(epcs.iloc[indA[0]]['name'])
rB = val['resp'].extract_epoch(epcs.iloc[indB[0]]['name'])
r_st.append(
full_resp.extract_epoch(epcs.iloc[ind]['name'])[:,
0, :])
rA_st_ = full_resp.extract_epoch(
epcs.iloc[indA[0]]['name'])[:, 0, :]
rB_st_ = full_resp.extract_epoch(
epcs.iloc[indB[0]]['name'])[:, 0, :]
rA_st.append(rA_st_)
rB_st.append(rB_st_)
minreps = np.min((rA_st_.shape[0], rB_st_.shape[0]))
rA_rB_st.append(rA_st_[:minreps, :] + rB_st_[:minreps, :])
if init:
rA_ = rA.squeeze()
rB_ = rB.squeeze()
r_ = r.squeeze()
rA_rB_ = rA.squeeze() + rB.squeeze()
init = False
else:
rA_ = np.hstack((rA_, rA.squeeze()))
rB_ = np.hstack((rB_, rB.squeeze()))
r_ = np.hstack((r_, r.squeeze()))
rA_rB_ = np.hstack(
(rA_rB_, rA.squeeze() + rB.squeeze()))
val['linmodel'] = val['linmodel'].replace_epoch(
epcs.iloc[ind]['name'], rA + rB, preserve_nan=False)
ff = np.isfinite(r_) & np.isfinite(rA_) & np.isfinite(
rB_) # find places with data
r_dual_A[_type] = np.corrcoef(rA_[ff], r_[ff])[
0, 1] # Correlation between response to A and response to dual
r_dual_B[_type] = np.corrcoef(rB_[ff], r_[ff])[
0, 1] # Correlation between response to B and response to dual
r_lin_A[_type] = np.corrcoef(
rA_[ff], rA_rB_[ff]
)[0,
1] # Correlation between response to A and response to linear 'model'
r_lin_B[_type] = np.corrcoef(
rB_[ff], rA_rB_[ff]
)[0,
1] # Correlation between response to B and response to linear 'model'
# correlations over single-trial data
minreps = np.min([x.shape[0] for x in r_st])
r_st = [x[:minreps, :] for x in r_st]
r_st = np.concatenate(r_st, axis=1)
rA_st = [x[:minreps, :] for x in rA_st]
rA_st = np.concatenate(rA_st, axis=1)
rB_st = [x[:minreps, :] for x in rB_st]
rB_st = np.concatenate(rB_st, axis=1)
rA_rB_st = [x[:minreps, :] for x in rA_rB_st]
rA_rB_st = np.concatenate(rA_rB_st, axis=1)
r_lin_A_bal[_type] = np.corrcoef(rA_st[0::2, ff].mean(axis=0),
rA_rB_st[1::2, ff].mean(axis=0))[0, 1]
r_lin_B_bal[_type] = np.corrcoef(rB_st[0::2, ff].mean(axis=0),
rA_rB_st[1::2, ff].mean(axis=0))[0, 1]
r_dual_A_bal[_type] = np.corrcoef(rA_st[0::2, ff].mean(axis=0),
r_st[:, ff].mean(axis=0))[0, 1]
r_dual_B_bal[_type] = np.corrcoef(rB_st[0::2, ff].mean(axis=0),
r_st[:, ff].mean(axis=0))[0, 1]
r_dual_A_nc[_type] = ohel.r_noise_corrected(rA_st, r_st)
r_dual_B_nc[_type] = ohel.r_noise_corrected(rB_st, r_st)
r_lin_A_nc[_type] = ohel.r_noise_corrected(rA_st, rA_rB_st)
r_lin_B_nc[_type] = ohel.r_noise_corrected(rB_st, rA_rB_st)
if _type == '11':
r11 = nems.metrics.corrcoef._r_single(r_st, 200, 0)
elif _type == '12':
r12 = nems.metrics.corrcoef._r_single(r_st, 200, 0)
elif _type == '21':
r21 = nems.metrics.corrcoef._r_single(r_st, 200, 0)
elif _type == '22':
r22 = nems.metrics.corrcoef._r_single(r_st, 200, 0)
# rac = _r_single(X, N)
# r_ceiling = [nmet.r_ceiling(p, rec, 'pred', 'resp') for p in val_copy]
# Things that used to happen only for _type is 'C' but still seem valid
r_A_B = np.corrcoef(rA_[ff], rB_[ff])[0, 1]
r_A_B_nc = r_noise_corrected(rA_st, rB_st)
rAA = nems.metrics.corrcoef._r_single(rA_st, 200, 0)
rBB = nems.metrics.corrcoef._r_single(rB_st, 200, 0)
Np = 0
rAA_nc = np.zeros(Np)
rBB_nc = np.zeros(Np)
hv = int(minreps / 2)
for i in range(Np):
inds = np.random.permutation(minreps)
rAA_nc[i] = sp.r_noise_corrected(rA_st[inds[:hv]], rA_st[inds[hv:]])
rBB_nc[i] = sp.r_noise_corrected(rB_st[inds[:hv]], rB_st[inds[hv:]])
ffA = np.isfinite(rAA_nc)
ffB = np.isfinite(rBB_nc)
rAAm = rAA_nc[ffA].mean()
rBBm = rBB_nc[ffB].mean()
mean_nsA = rA_st.sum(axis=1).mean()
mean_nsB = rB_st.sum(axis=1).mean()
min_nsA = rA_st.sum(axis=1).min()
min_nsB = rB_st.sum(axis=1).min()
# Calculate correlation between linear 'model and dual-voice response, and mean amount of suppression, enhancement relative to linear 'model'
r_fit_linmodel = {}
r_fit_linmodel_NM = {}
r_ceil_linmodel = {}
mean_enh = {}
mean_supp = {}
EnhP = {}
SuppP = {}
DualAboveZeroP = {}
resp_ = copy.deepcopy(rec['resp'].rasterize())
resp_.epochs['start'] = sts_rec
fn = lambda x: np.atleast_2d(
sp.smooth(x.squeeze(), 3, 2) - SR / val['resp'].fs)
resp_ = resp_.transform(fn)
for _type in types:
val_copy = copy.deepcopy(val)
# from pdb import set_trace
# set_trace()
val_copy['resp'] = val_copy['resp'].select_epochs([_type])
# Correlation between linear 'model' (response to A plus response to B) and dual-voice response
r_fit_linmodel_NM[_type] = nmet.corrcoef(val_copy, 'linmodel', 'resp')
# r_ceil_linmodel[_type] = nems.metrics.corrcoef.r_ceiling(val_copy,rec,'linmodel', 'resp',exclude_neg_pred=False)[0]
# Noise-corrected correlation between linear 'model' (response to A plus response to B) and dual-voice response
r_ceil_linmodel[_type] = nems.metrics.corrcoef.r_ceiling(
val_copy, rec, 'linmodel', 'resp')[0]
pred = val_copy['linmodel'].as_continuous()
resp = val_copy['resp'].as_continuous()
ff = np.isfinite(pred) & np.isfinite(resp)
# cc = np.corrcoef(sp.smooth(pred[ff],3,2), sp.smooth(resp[ff],3,2))
cc = np.corrcoef(pred[ff], resp[ff])
r_fit_linmodel[_type] = cc[0, 1]
prdiff = resp[ff] - pred[ff]
mean_enh[_type] = prdiff[prdiff > 0].mean() * val['resp'].fs
mean_supp[_type] = prdiff[prdiff < 0].mean() * val['resp'].fs
# Find percent of time response is suppressed vs enhanced relative to what would be expected by a linear sum of single-voice responses
# First, jacknife to find...
# Njk=10
# if _type is 'C':
# stims=['STIM_T+si464+si464','STIM_T+si516+si516']
# else:
# stims=['STIM_T+si464+si516', 'STIM_T+si516+si464']
# T=int(700+prestim*val['resp'].fs)
# Tps=int(prestim*val['resp'].fs)
# jns=np.zeros((Njk,T,len(stims)))
# for ns in range(len(stims)):
# for njk in range(Njk):
# resp_jn=resp_.jackknife_by_epoch(Njk,njk,stims[ns])
# jns[njk,:,ns]=np.nanmean(resp_jn.extract_epoch(stims[ns]),axis=0)
# jns=np.reshape(jns[:,Tps:,:],(Njk,700*len(stims)),order='F')
#
# lim_models=np.zeros((700,len(stims)))
# for ns in range(len(stims)):
# lim_models[:,ns]=val_copy['linmodel'].extract_epoch(stims[ns])
# lim_models=lim_models.reshape(700*len(stims),order='F')
#
# ff=np.isfinite(lim_models)
# mean_diff=(jns[:,ff]-lim_models[ff]).mean(axis=0)
# std_diff=(jns[:,ff]-lim_models[ff]).std(axis=0)
# serr_diff=np.sqrt(Njk/(Njk-1))*std_diff
#
# thresh=3
# dual_above_zero = (jns[:,ff].mean(axis=0) > std_diff)
# sig_enh = ((mean_diff/serr_diff) > thresh) & dual_above_zero
# sig_supp = ((mean_diff/serr_diff) < -thresh)
# DualAboveZeroP[_type] = (dual_above_zero).sum()/len(mean_diff)
# EnhP[_type] = (sig_enh).sum()/len(mean_diff)
# SuppP[_type] = (sig_supp).sum()/len(mean_diff)
# time = np.arange(0, lim_models.shape[0])/ val['resp'].fs
# plt.figure();
# plt.plot(time,jns.mean(axis=0),'.-k');
# plt.plot(time,lim_models,'.-g');
# plt.plot(time[sig_enh],lim_models[sig_enh],'.r')
# plt.plot(time[sig_supp],lim_models[sig_supp],'.b')
# plt.title('Type:{:s}, Enh:{:.2f}, Sup:{:.2f}, Resp_above_zero:{:.2f}'.format(_type,EnhP[_type],SuppP[_type],DualAboveZeroP[_type]))
# from pdb import set_trace
# set_trace()
# a=2
# thrsh=5
# EnhP[_type] = ((prdiff*val['resp'].fs) > thresh).sum()/len(prdiff)
# SuppP[_type] = ((prdiff*val['resp'].fs) < -thresh).sum()/len(prdiff)
# return val
# return {'excitatory_percentage':excitatory_percentage,
# 'inhibitory_percentage':inhibitory_percentage,
# 'r_fit_linmodel':r_fit_linmodel,
# 'SR':SR, 'SR_std':SR_std, 'SR_av_std':SR_av_std}
#
return {
'thresh': thresh * val['resp'].fs,
'EP_A': excitatory_percentage['A'],
'EP_B': excitatory_percentage['B'],
# 'EP_C':excitatory_percentage['C'],
'EP_I': excitatory_percentage['I'],
'IP_A': inhibitory_percentage['A'],
'IP_B': inhibitory_percentage['B'],
# 'IP_C':inhibitory_percentage['C'],
'IP_I': inhibitory_percentage['I'],
'OEP_A': excitatory_percentage_onset['A'],
'OEP_B': excitatory_percentage_onset['B'],
# 'OEP_C':excitatory_percentage_onset['C'],
'OEP_I': excitatory_percentage_onset['I'],
'OIP_A': inhibitory_percentage_onset['A'],
'OIP_B': inhibitory_percentage_onset['B'],
# 'OIP_C':inhibitory_percentage_onset['C'],
'OIP_I': inhibitory_percentage_onset['I'],
'Max_A': Max['A'],
'Max_B': Max['B'],
# 'Max_C':Max['C'],
'Max_I': Max['I'],
'Mean_A': Mean['A'],
'Mean_B': Mean['B'],
# 'Mean_C':Mean['C'],
'Mean_I': Mean['I'],
'OMax_A': Max_onset['A'],
'OMax_B': Max_onset['B'],
# 'OMax_C':Max_onset['C'],
'OMax_I': Max_onset['I'],
'TotalMax': TotalMax * val['resp'].fs,
'SinglesMax': SinglesMax * val['resp'].fs,
# 'r_lin_C':r_fit_linmodel['C'],
'r_lin_I': r_fit_linmodel['I'],
# 'r_lin_C_NM':r_fit_linmodel_NM['C'],
'r_lin_I_NM': r_fit_linmodel_NM['I'],
# 'r_ceil_C':r_ceil_linmodel['C'],
'r_ceil_I': r_ceil_linmodel['I'],
# 'MEnh_C':mean_enh['C'],
'MEnh_I': mean_enh['I'],
# 'MSupp_C':mean_supp['C'],
'MSupp_I': mean_supp['I'],
# 'EnhP_C':EnhP['C'],
# 'EnhP_I':EnhP['I'],
# 'SuppP_C':SuppP['C'],
# 'SuppP_I':SuppP['I'],
# 'DualAboveZeroP_C':DualAboveZeroP['C'],
# 'DualAboveZeroP_I':DualAboveZeroP['I'],
# 'r_dual_A_C':r_dual_A['C'],
'r_dual_A_I': r_dual_A['I'],
# 'r_dual_B_C':r_dual_B['C'],
'r_dual_B_I': r_dual_B['I'],
# 'r_dual_A_C_nc':r_dual_A_nc['C'],
'r_dual_A_I_nc': r_dual_A_nc['I'],
# 'r_dual_B_C_nc':r_dual_B_nc['C'],
'r_dual_B_I_nc': r_dual_B_nc['I'],
# 'r_dual_A_C_bal':r_dual_A_bal['C'],
'r_dual_A_I_bal': r_dual_A_bal['I'],
# 'r_dual_B_C_bal':r_dual_B_bal['C'],
'r_dual_B_I_bal': r_dual_B_bal['I'],
# 'r_lin_A_C':r_lin_A['C'],
'r_lin_A_I': r_lin_A['I'],
# 'r_lin_B_C':r_lin_B['C'],
'r_lin_B_I': r_lin_B['I'],
# 'r_lin_A_C_nc':r_lin_A_nc['C'],
'r_lin_A_I_nc': r_lin_A_nc['I'],
# 'r_lin_B_C_nc':r_lin_B_nc['C'],
'r_lin_B_I_nc': r_lin_B_nc['I'],
# 'r_lin_A_C_bal':r_lin_A_bal['C'],
'r_lin_A_I_bal': r_lin_A_bal['I'],
# 'r_lin_B_C_bal':r_lin_B_bal['C'],
'r_lin_B_I_bal': r_lin_B_bal['I'],
'r_A_B': r_A_B,
'r_A_B_nc': r_A_B_nc,
'rAAm': rAAm,
'rBBm': rBBm,
'rAA': rAA,
'rBB': rBB,
'rII': rII,
# 'rCC':rCC
'rAA_nc': rAA_nc,
'rBB_nc': rBB_nc,
'mean_nsA': mean_nsA,
'mean_nsB': mean_nsB,
'min_nsA': min_nsA,
'min_nsB': min_nsB,
'SR': SR,
'SR_std': SR_std,
'SR_av_std': SR_av_std,
'norm_spont': norm_spont,
'spont_rate': spont_rate,
'params': params,
'corcoef': corcoef,
'avg_resp': avg_resp,
'snr': snr,
'pair_names': twostims,
'suppression': supp_array,
'FR': FR_array,
'rec': rec,
'animal': cellid[:3]
}
# siteids
sites = ['CRD009b', 'CRD010b', 'CRD011c', 'CRD012b', 'CRD013b', 'CRD016c', 'CRD017c', 'CRD018d', 'CRD019b']
for site in sites:
if os.path.isdir(os.path.join(fig_path, site)):
pass
else:
os.mkdir(os.path.join(fig_path, site))
site_path = os.path.join(fig_path, site)
# get parmfiles
sql = "SELECT sCellFile.cellid, sCellFile.respfile, gDataRaw.resppath from sCellFile INNER JOIN" \
" gCellMaster ON (gCellMaster.id=sCellFile.masterid) INNER JOIN" \
" gDataRaw ON (sCellFile.rawid=gDataRaw.id)" \
" WHERE gCellMaster.siteid=%s" \
" and gDataRaw.runclass='TBP' and gDataRaw.bad=0"
d = nd.pd_query(sql, (site,))
d['parmfile'] = [f.replace('.spk.mat', '.m') for f in d['respfile']]
parmfiles = np.unique(np.sort([os.path.join(d['resppath'].iloc[i], d['parmfile'].iloc[i]) for i in range(d.shape[0])])).tolist()
manager = BAPHYExperiment(parmfiles)
rec = manager.get_recording(**options)
rec['resp'] = rec['resp'].rasterize()
# find / sort epoch names
files = [f for f in rec['resp'].epochs.name.unique() if 'FILE_' in f]
targets = [f for f in rec['resp'].epochs.name.unique() if 'TAR_' in f]
catch = [f for f in rec['resp'].epochs.name.unique() if 'CAT_' in f]
sounds = targets + catch
ref_stims = [x for x in rec['resp'].epochs.name.unique() if 'STIM_' in x]
idx = np.argsort([int(s.split('_')[-1]) for s in ref_stims])
ref_stims = np.array(ref_stims)[idx].tolist()
# ======================================================================
dfpath = '/auto/users/hellerc/code/projects/rewardLearning/R01_renewal_figs/results/'
runclass = 'BVT'
animal = 'Cordyceps'
earliest_date = '2019_11_09'
#animal = 'Drechsler'
#earliest_date = '2019_10_14'
an_regex = "%"+animal+"%"
min_trials = 50
# get list of all training parmfiles
sql = "SELECT parmfile, resppath, gDataRaw.id, cellid, gData.svalue, gData.value FROM gDataRaw INNER JOIN gData on (gDataRaw.id=gData.rawid and gData.name='Tar_Frequencies') WHERE runclass=%s and resppath like %s and training = 1 and bad=0 and trials>%s and behavior='active'"
parmfiles1 = nd.pd_query(sql, (runclass, an_regex, min_trials))
parmfiles1 = parmfiles1.set_index('id')
sql = "SELECT parmfile, resppath, gDataRaw.id, cellid, gData.svalue FROM gDataRaw INNER JOIN gData on (gDataRaw.id=gData.rawid and gData.name='Behave_PumpDuration') WHERE runclass=%s and resppath like %s and training = 1 and bad = 0 and trials>%s and behavior='active'"
parmfiles = nd.pd_query(sql, (runclass, an_regex, min_trials))
parmfiles = parmfiles.set_index('id')
parmfiles = parmfiles.rename(columns={'svalue': 'pumpdur'})
parmfiles = pd.concat([parmfiles['pumpdur'], parmfiles1], axis=1, join='outer')
# screen for dates
parmfiles['date'] = [dt.datetime.strptime('-'.join(x.split('_')[1:-2]), '%Y-%m-%d') for x in parmfiles.parmfile]
ed = dt.datetime.strptime(earliest_date, '%Y_%m_%d')
parmfiles = parmfiles[parmfiles.date > ed]
options = {'pupil': False, 'rasterfs': 100}
def mark_complete(pupilfiles):
"""
Save predictions for all files and update celldb to mark these analyses as complete.
!!!!!!!!!!!!!!!! Take care using this!!!!!!!!!!!!!!!!!!!!!!!!!!!!
It's a good idea to first use pupil_browser to validate the results
of the fit. However, if you collect many videos in a row and the hardware set up
doesn't change, it's likely you can just perform the pupil_browser QC on the first
video in the sequence and then batch process the rest of the videos using this function.
"""
for vf in pupilfiles:
video_name = os.path.splitext(os.path.split(vf)[-1])[0]
fn = video_name + '.pickle'
fn_mat = video_name + '.mat'
fp = os.path.split(vf)[0]
save_path = os.path.join(fp, 'sorted', fn)
# for matlab loading
mat_fn = os.path.join(fp, fn_mat)
sorted_dir = os.path.split(save_path)[0]
if os.path.isdir(sorted_dir) != True:
# create sorted directory and force to be world writeable
os.system("mkdir {}".format(sorted_dir))
os.system("chmod a+w {}".format(sorted_dir))
print("created new directory {0}".format(sorted_dir))
else:
pass
try:
# load predictions
pred_path = os.path.join(sorted_dir, fn.replace('.pickle', '_pred.pickle'))
with open(pred_path, 'rb') as fp:
save_dict = pickle.load(fp)
# No excluded frames options for batch saving
save_dict['cnn']['excluded_frames'] = []
x_diff = np.diff(save_dict['cnn']['x'])
y_diff = np.diff(save_dict['cnn']['y'])
d = np.sqrt((x_diff ** 2) + (y_diff ** 2))
d[-1] = 0
d = np.concatenate((d, np.zeros(1)))
save_dict['cnn']['eyespeed'] = d
with open(save_path, 'wb') as fp:
pickle.dump(save_dict, fp, protocol=pickle.HIGHEST_PROTOCOL)
scipy.io.savemat(mat_fn, save_dict)
# finally, update celldb to mark pupil as analyzed
get_file1 = "SELECT eyecalfile from gDataRaw where eyecalfile LIKE %s"
out1 = nd.pd_query(get_file1, params=("%" + vf.replace('/auto/data/daq/', '') + "%",))
og_video_path = out1.iloc[0][0]
sql = "UPDATE gDataRaw SET eyewin=2 WHERE eyecalfile='{}'".format(og_video_path)
nd.sql_command(sql)
log.info("Saved analysis successfully for {}".format(video_name))
except:
log.info("No saved pupil analysis {}".format(os.path.join(sorted_dir, fn.replace('.pickle', '_pred.pickle'))))
