def dPrime_from_NIT_site(site, duration, source, position, meta):
options = {
'batch': batch,
'siteid': site,
'stimfmt': 'envelope',
'rasterfs': load_fs,
'recache': False,
'runclass': 'NTI',
'stim': False
}
load_URI = nb.baphy_load_recording_uri(**options)
rec = recording.load_recording(load_URI)
rec = set_recording_subepochs(rec)
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
probe_regex = NTI_epoch_name(duration, source, position)
cp_regex = fr'\AC(({NTI_epoch_name()})|(PreStimSilence))_P{probe_regex}\Z'
full_rast, transitions, contexts = raster_from_sig(
sig, cp_regex, goodcells, raster_fs=meta['raster_fs'])
if len(contexts) < 2:
real = shuffled = simulated = None
print(f'only one context for {probe_regex}, skiping analysis')
else:
real, shuffled, simulated = nway_analysis(full_rast, meta)
return real, shuffled, simulated, transitions, contexts
def load(site, boxload=True, **kwargs):
# defaults
options = {'batch': 316,
'cellid': site,
'stimfmt': 'envelope',
'rasterfs': 100,
'recache': False,
'runclass': 'CPN',
'stim': False}
options.update(**kwargs)
if boxload is True:
toname = options.copy()
del toname['recache']
filename = pl.Path(config['paths']['recording_cache']) / set_name(toname)
if not filename.parent.exists():
filename.parent.mkdir()
if filename.exists() and options['recache'] is False:
print('loading recording from box')
loaded_rec = jl.load(filename)
elif filename.exists() is False or options['recache'] is True:
load_URI, _ = nb.baphy_load_recording_uri(**options)
loaded_rec = recording.load_recording(load_URI)
print('cacheing recoring in box')
jl.dump(loaded_rec, filename)
else:
raise SystemError('WTF?')
elif boxload is False:
load_URI, _ = nb.baphy_load_recording_uri(**options)
loaded_rec = recording.load_recording(load_URI)
else:
raise ValueError('boxload must be boolean')
CPN_rec = cpe.set_recording_subepochs(loaded_rec)
recordings = split_recording(CPN_rec)
return recordings
# If using database:
batch = 289 # NAT + pupil
cellid = 'BRT036b-45-2'
#cellid = 'BRT037b-63-1'
cellid = 'TAR010c-13-1'
cellid = 'TAR009d-42-1'
options = {
'rasterfs': 100,
'stimfmt': 'ozgf',
'chancount': 18,
'pupil': True,
'stim': True
}
# get the name of the cached recording
uri = nb.baphy_load_recording_uri(cellid=cellid, batch=batch, **options)
rec = load_recording(uri)
# convert to rasterized signals from PointProcess and TiledSignal
rec['resp'] = rec['resp'].rasterize()
rec['stim'] = rec['stim'].rasterize()
rec['resp'] = rec['resp'].extract_channels([cellid])
rec.meta["cellid"] = cellid
#est, val = estimation, validation data sets
est, val = rec.split_using_epoch_occurrence_counts(epoch_regex="^STIM_")
est = average_away_epoch_occurrences(est, epoch_regex="^STIM_")
val = average_away_epoch_occurrences(val, epoch_regex="^STIM_")
# get matrices for fitting:
fn = '/auto/users/hellerc/code/projects/Cosyne2020_poster/svgs/dec_cv_example.svg'
site = 'TAR010c'
nPCs = 4
batch = 307
fs = 20
rawid = pu.which_rawids(site)
ops = {
'batch': batch,
'pupil': 1,
'rasterfs': fs,
'siteid': site,
'stim': 0,
'rawid': rawid
}
uri = nb.baphy_load_recording_uri(**ops)
rec = Recording.load(uri)
rec['resp'] = rec['resp'].rasterize()
rec = rec.and_mask(['HIT_TRIAL', 'MISS_TRIAL', 'PASSIVE_EXPERIMENT'])
rec = rec.and_mask(['PreStimSilence', 'PostStimSilence'], invert=True)
rec = rec.apply_mask(reset_epochs=True)
# create four state masks
rec = preproc.create_ptd_masks(rec)
# create the appropriate dictionaries of responses
# don't *think* it's so important to balanced reps for
# this analysis since it's parametric (as opposed to the
# discrimination calc which depends greatly on being balanced)
def generate_recording_uri(cellid=None, batch=None, loadkey=None,
siteid=None, **options):
"""
required parameters (passed through to nb.baphy_data_path):
cellid: string or list
string can be a valid cellid or siteid
list is a list of cellids from a single(?) site
batch: integer
figure out filename (or eventually URI) of pre-generated
NEMS-format recording for a given cell/batch/loader string
very baphy-specific. Needs to be coordinated with loader processing
in nems.xform_helper
"""
# remove any preprocessing keywords in the loader string.
loader = nems.utils.escaped_split(loadkey, '-')[0]
log.info('loader=%s',loader)
ops = loader.split(".")
# updates some some defaults
options.update({'rasterfs': 100, 'chancount': 0})
load_pop_file = False
for op in ops:
if op=='ozgf':
options['stimfmt'] = 'ozgf'
elif op=='parm':
options['stimfmt'] = 'parm'
elif op=='env':
options['stimfmt'] = 'envelope'
elif op in ['nostim','psth','ns', 'evt']:
options.update({'stim': False, 'stimfmt': 'parm'})
elif op.startswith('fs'):
options['rasterfs'] = int(op[2:])
elif op.startswith('ch'):
options['chancount'] = int(op[2:])
elif op=='pup':
options.update({'pupil': True, 'pupil_deblink': True,
'pupil_deblink_dur': 1,
'pupil_median': 0})
elif op=='rem':
options['rem'] = True
elif 'eysp' in ops:
options['pupil_eyespeed'] = True
elif op.startswith('pop'):
load_pop_file = True
if 'stimfmt' not in options.keys():
raise ValueError('Valid stim format (ozgf, psth, parm, env, evt) not specified in loader='+loader)
if (options['stimfmt']=='ozgf') and (options['chancount'] <= 0):
raise ValueError('Stim format ozgf requires chancount>0 (.chNN) in loader='+loader)
if int(batch) == 294:
options["runclass"] = "VOC"
if siteid is not None:
options['siteid'] = siteid
options["batch"] = batch
options["cellid"] = cellid
if load_pop_file:
recording_uri = pop_file(siteid=cellid, **options)
else:
recording_uri = nb.baphy_load_recording_uri(**options)
return recording_uri
ax_lab_size = 15
ax_val_size = 11
########################################################################################################################
site = 'AMT028b' # example site
options = {
'batch': batch,
'siteid': site,
'stimfmt': 'envelope',
'rasterfs': 100,
'recache': False,
'runclass': 'NTI',
'stim': False
}
load_URI = nb.baphy_load_recording_uri(**options)
rec = recording.load_recording(load_URI)
rec = set_recording_subepochs(rec)
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
######### generates an array ordering a probe and its contexts. Defines the kind of transitions for such cp combinations
# [C, P, R, U, T] final array dimesions
# information of what set of sequences pairs to use
def kamiak_array(cellids, batch, modelnames, output_path):
# Put in folder for today's date
subdirectory = str(datetime.datetime.now()).split(' ')[0]
directory_path = os.path.join(output_path, subdirectory)
if not os.path.exists(directory_path):
os.makedirs(directory_path, exist_ok=True)
# Create a manifest of the recording names needed
manifest_path = os.path.join(directory_path, 'manifest.sh')
reverse_manifest_path = os.path.join(directory_path, 'reverse_manifest.sh')
args_path = os.path.join(directory_path, 'jobs.txt')
script_path = os.path.join(directory_path, 'batch.srun')
recording_entries = []
args_entries = []
remote_host = "*****@*****.**"
recordings = "/home/jacob.pennington/nems/recordings/"
results = f"/home/jacob.pennington/nems/results/{batch}"
remote_recordings = f"{remote_host}:{recordings}"
remote_results = f"{remote_host}:{results}"
scripts = "/home/jacob.pennington/slurm_scripts/"
remote_scripts = f"{remote_host}:{scripts}"
logs = "/home/jacob.pennington/nems_logs/"
jobs = f"{scripts}/{subdirectory}/jobs.txt"
failed_jobs = jobs[:-4] + '_failed.txt'
for j, c in enumerate(cellids):
# TODO: Don't hardcode the loader options, parse from modelname
options = {
'cellid': c,
'batch': batch,
'stim': 1,
'stimfmt': 'ozgf',
'chancount': 18,
'rasterfs': 100
}
# Record unique recording URIs (may be shared for c with same siteid)
recording_uri = nb.baphy_load_recording_uri(**options)
if recording_uri not in recording_entries:
recording_entries.append(recording_uri)
remote_rec = f'{recordings}/{recording_uri.split("/")[-1]}'
for i, m in enumerate(modelnames):
args = f"{c} {batch} {m} {remote_rec} {recording_uri}"
args_entries.append(args)
script_contents = (
"#!/bin/bash\n"
"#SBATCH --partition=kamiak\n"
"#SBATCH --job-name=NEMS\n"
f"#SBATCH --array=1-{len(args_entries)}\n"
f"#SBATCH --output={logs}/{subdirectory}/NEMS.%A_%a.out\n"
f"#SBATCH --error={logs}/{subdirectory}/NEMS.%A_%a.err\n"
"#SBATCH --time=1-23:59:00\n"
"#SBATCH --nodes=1\n"
"#SBATCH --ntasks-per-node=1\n"
"\n"
f"job=$(sed \"${{SLURM_ARRAY_TASK_ID}}q;d\" {jobs})\n"
"args=($job)\n"
"\n"
"# Call cleanup function on cancelled jobs\n"
"function clean_up {\n"
" echo \"${args[0]} ${args[1]} ${args[2]}"
f" ${{args[3]}} ${{args[4]}}\" >> {failed_jobs}\n"
"}\n"
"# trap termination signals\n"
"trap 'clean_up' SIGINT SIGTERM\n"
"python3 /home/jacob.pennington/nems_scripts/fit_xforms.py "
"${args[0]} ${args[1]} ${args[2]} ${args[3]} ${args[4]}\n"
"echo \"task ${SLURM_ARRAY_TASK_ID} complete\"")
# Write recording_uri list to manifest
# Copy recording files in separate scp commands since
# a full batch might become very large for a single copy
manifest_lines = [
f'rsync -avx --ignore-existing {e} {remote_recordings}/{e.split("/")[-1]}'
for e in recording_entries
]
# But all scripts can be copied over in one command
manifest_lines.append(f'scp {args_path} {remote_scripts}/{subdirectory}/')
manifest_lines.append(
f'scp {script_path} {remote_scripts}/{subdirectory}/')
manifest_lines.insert(
0, f'ssh {remote_host} "mkdir -p {scripts}/{subdirectory}"')
manifest_lines.insert(
0, f'ssh {remote_host} "mkdir -p {logs}/{subdirectory}"')
manifest_lines.insert(0, '#!/bin/bash')
manifest_contents = '\n'.join([e for e in manifest_lines])
reverse_manifest_lines = [f'rsync -avx {remote_results} $1']
reverse_manifest_lines.insert(0, '#!/bin/bash')
reverse_manifest_contents = '\n'.join([r for r in reverse_manifest_lines])
args_contents = '\n'.join([a for a in args_entries])
with open(manifest_path, 'w+') as manifest:
manifest.write(manifest_contents)
os.chmod(manifest_path, 0o777) # open up permissions for other users
st = os.stat(manifest_path)
os.chmod(manifest_path, st.st_mode | stat.S_IEXEC) # make it executable
with open(reverse_manifest_path, 'w+') as reverse_manifest:
reverse_manifest.write(reverse_manifest_contents)
os.chmod(reverse_manifest_path, 0o777)
rst = os.stat(reverse_manifest_path)
os.chmod(reverse_manifest_path, rst.st_mode | stat.S_IEXEC)
with open(args_path, 'w+') as args:
args.write(args_contents)
with open(script_path, 'w+') as script:
script.write(script_contents)
def kamiak_batch(cellids, batch, modelnames, output_path):
# Put in folder for today's date
subdirectory = str(datetime.datetime.now()).split(' ')[0]
directory_path = os.path.join(output_path, subdirectory)
if not os.path.exists(directory_path):
os.makedirs(directory_path, exist_ok=True)
# Create a manifest of the recording names needed
manifest_path = os.path.join(directory_path, 'manifest.sh')
recording_entries = []
script_entries = []
remote_host = "*****@*****.**"
recordings = "/home/jacob.pennington/nems/recordings/"
remote_recordings = f"{remote_host}:{recordings}"
scripts = "/home/jacob.pennington/slurm_scripts/"
remote_scripts = f"{remote_host}:{scripts}"
for j, c in enumerate(cellids):
# TODO: Don't hardcode the loader options, parse from modelname
options = {
'cellid': c,
'batch': batch,
'stim': 1,
'stimfmt': 'ozgf',
'chancount': 18,
'rasterfs': 100
}
# Record unique recording URIs (may be shared for c with same siteid)
recording_uri = nb.baphy_load_recording_uri(**options)
if recording_uri not in recording_entries:
recording_entries.append(recording_uri)
remote_rec = f'{recordings}/{recording_uri.split("/")[-1]}'
for i, m in enumerate(modelnames):
name = f'{m}__{batch}__{c}'
filename = f'{name}.srun'
jobname = f'NEMS_model{i}_cell{j}'
full_path = os.path.join(directory_path, filename)
contents = (
"#!/bin/bash\n"
"#SBATCH --partition=kamiak\n"
f"#SBATCH --job-name={jobname}\n"
f"#SBATCH --output=/home/jacob.pennington/nems_logs/{subdirectory}/{name}.out\n"
f"#SBATCH --error=/home/jacob.pennington/nems_logs/{subdirectory}/{name}.err\n"
"#SBATCH --time=1-23:59:00\n"
"#SBATCH --nodes=1\n"
"#SBATCH --ntasks-per-node=1\n"
"python3 /home/jacob.pennington/nems_scripts/fit_xforms.py "
f"'{c}' '{batch}' '{m}' '{remote_rec}' '{recording_uri}'")
with open(full_path, 'w+') as script:
script.write(contents)
script_entries.append(full_path)
# Write recording_uri list to manifest
# Copy recording files in separate scp commands since
# a full batch might become very large for a single copy
manifest_lines = [
f'scp {e} {remote_recordings}/{e.split("/")[-1]}'
for e in recording_entries
]
# But all scripts can be copied over in one command
all_scripts = ' '.join(script_entries)
manifest_lines += [f'scp {all_scripts} {remote_scripts}/{subdirectory}/']
manifest_lines.insert(
0, f'ssh {remote_host} "mkdir -p {scripts}/{subdirectory}"')
manifest_lines.insert(0, '#!/bin/bash')
manifest_contents = '\n'.join([str(e) for e in manifest_lines])
with open(manifest_path, 'w+') as manifest:
manifest.write(manifest_contents)
st = os.stat(manifest_path)
os.chmod(manifest_path, st.st_mode | stat.S_IEXEC) # make it executable
def plot_collapsed_ref_tar(animal, site, cellids=None):
site += "%"
sql = "SELECT DISTINCT cellid, rawid, respfile FROM sCellFile WHERE cellid like %s AND runclassid=%s"
d = nd.pd_query(sql, params=(
site,
42,
))
mfile = []
for f in np.unique(d.respfile):
f_ = f.split('.')[0]
mfile.append('/auto/data/daq/{0}/{1}/{2}'.format(
animal, site[:-2], f_))
if cellids is None:
cellid = np.unique(d.cellid).tolist()
else:
cellid = cellids
options = {
"siteid": site[:-1],
'cellid': cellids,
"mfilename": mfile,
'stim': False,
'pupil': True,
'rasterfs': 1000
}
uri = nb.baphy_load_recording_uri(**options)
rec = Recording.load(uri)
all_pupil = rec['pupil']._data
ncols = len(mfile)
if cellids is None:
cellids = rec['resp'].chans
for c in cellids:
f, ax = plt.subplots(1, ncols, sharey=True, figsize=(12, 5))
ref_base = 0
for i, mf in enumerate(mfile):
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
R = rec.copy()
R['resp'] = R['resp'].rasterize(fs=20)
R['resp'].fs = 20
R = R.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
R = R.and_mask(['HIT_TRIAL']).apply_mask(reset_epochs=True)
resp = R['resp'].extract_channels([c])
tar_reps = resp.extract_epoch('TARGET').shape[0]
tar_m = np.nanmean(resp.extract_epoch('TARGET'),
0).squeeze() * R['resp'].fs
tar_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('TARGET'),
0).squeeze() / np.sqrt(tar_reps)
ref_reps = resp.extract_epoch('REFERENCE').shape[0]
ref_m = np.nanmean(resp.extract_epoch('REFERENCE'),
0).squeeze() * R['resp'].fs
ref_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('REFERENCE'),
0).squeeze() / np.sqrt(ref_reps)
# plot psth's
time = np.linspace(0, len(tar_m) / R['resp'].fs, len(tar_m))
ax[i].plot(time, tar_m, color='red', lw=2)
ax[i].fill_between(time,
tar_m + tar_sem,
tar_m - tar_sem,
color='coral')
time = np.linspace(0, len(ref_m) / R['resp'].fs, len(ref_m))
ax[i].plot(time, ref_m, color='blue', lw=2)
ax[i].fill_between(time,
ref_m + ref_sem,
ref_m - ref_sem,
color='lightblue')
# set title
ax[i].set_title(fn, fontsize=8)
# set labels
ax[i].set_xlabel('Time (s)')
ax[i].set_ylabel('Spk / sec')
# get raster plot baseline
base = np.max(np.concatenate((tar_m + tar_sem, ref_m + ref_sem)))
if base > ref_base:
ref_base = base
for i, mf in enumerate(mfile):
# plot the rasters
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
rast_rec = rec.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
rast_rec = rast_rec.and_mask(['HIT_TRIAL'
]).apply_mask(reset_epochs=True)
rast = rast_rec['resp'].extract_channels([c])
ref_times = np.where(rast.extract_epoch('REFERENCE').squeeze())
base = ref_base
ref_pupil = np.nanmean(
rast_rec['pupil'].extract_epoch('REFERENCE'), -1)
xoffset = rast.extract_epoch(
'TARGET').shape[-1] / rec['resp'].fs + 0.01
ax[i].plot(ref_pupil / np.max(all_pupil) + xoffset,
np.linspace(base, int(base * 2), len(ref_pupil)),
color='k')
ax[i].axvline(xoffset + np.median(all_pupil / np.max(all_pupil)),
linestyle='--',
color='lightgray')
#import pdb; pdb.set_trace()
if ref_times[0].size != 0:
max_rep = ref_pupil.shape[0] - 1
ref_locs = ref_times[0] * (base / max_rep)
ref_locs = ref_locs + base
ax[i].plot(ref_times[1] / rec['resp'].fs,
ref_locs,
'|',
color='blue',
markersize=1)
tar_times = np.where(rast.extract_epoch('TARGET').squeeze())
tar_pupil = np.nanmean(rast_rec['pupil'].extract_epoch('TARGET'),
-1)
tar_base = np.max(ref_locs) + 1
ax[i].plot(tar_pupil / np.max(all_pupil) + xoffset,
np.linspace(tar_base, tar_base + base, len(tar_pupil)),
color='k')
if tar_times[0].size != 0:
max_rep = tar_pupil.shape[0] - 1
tar_locs = tar_times[0] * (base / max_rep)
tar_locs = tar_locs + tar_base
ax[i].plot(tar_times[1] / rec['resp'].fs,
tar_locs,
'|',
color='red',
markersize=1)
# set ylim
#ax[i].set_ylim((0, top))
# set plot aspect
#asp = np.diff(ax[i].get_xlim())[0] / np.diff(ax[i].get_ylim())[0]
#ax[i].set_aspect(asp / 2)
f.suptitle(c, fontsize=8)
f.tight_layout()
def generate_recording_uri(cellid=None,
batch=None,
loadkey="",
siteid=None,
force_old_loader=False,
**options):
"""
required parameters (passed through to nb.baphy_data_path):
cellid: string or list
string can be a valid cellid or siteid
list is a list of cellids from a single(?) site
batch: integer
figure out filename (or eventually URI) of pre-generated
NEMS-format recording for a given cell/batch/loader string
very baphy-specific. Needs to be coordinated with loader processing
in nems.xform_helper
"""
# remove any preprocessing keywords in the loader string.
if '-' in loadkey:
loader = nems.utils.escaped_split(loadkey, '-')[0]
else:
loader = loadkey
if loader != '':
log.info('loader=%s', loader)
options = baphy_io.parse_loadkey(loadkey=loader,
batch=batch,
siteid=siteid,
cellid=cellid,
**options)
""" moved functionality to baphy_io
ops = loader.split(".")
# updates some some defaults
options['rasterfs'] = options.get('rasterfs', 100)
options['chancount'] = options.get('chancount', 0)
load_pop_file = False
for op in ops:
if op=='ozgf':
options['stimfmt'] = 'ozgf'
elif op=='parm':
options['stimfmt'] = 'parm'
elif op=='ll':
options['stimfmt'] = 'll'
elif op=='env':
options['stimfmt'] = 'envelope'
elif op in ['nostim','psth','ns', 'evt']:
options.update({'stim': False, 'stimfmt': 'parm'})
elif op.startswith('fs'):
options['rasterfs'] = int(op[2:])
elif op.startswith('ch'):
options['chancount'] = int(op[2:])
elif op.startswith('fmap'):
options['facemap'] = int(op[4:])
elif op=='pup':
options.update({'pupil': True, 'rem': 1})
#options.update({'pupil': True, 'pupil_deblink': True,
# 'pupil_deblink_dur': 1,
# 'pupil_median': 0, 'rem': 1})
elif op=='dlc':
options.update({'dlc': True})
elif op=='rem':
options['rem'] = True
elif 'eysp' in ops:
options['pupil_eyespeed'] = True
elif op.startswith('pop'):
load_pop_file = True
elif op == 'voc':
options.update({'runclass': 'VOC'})
if 'stimfmt' not in options.keys():
raise ValueError('Valid stim format (ozgf, psth, parm, env, evt) not specified in loader='+loader)
if (options['stimfmt']=='ozgf') and (options['chancount'] <= 0):
raise ValueError('Stim format ozgf requires chancount>0 (.chNN) in loader='+loader)
if int(batch) in [263,294]:
options["runclass"] = "VOC"
if siteid is not None:
options['siteid'] = siteid
options["batch"] = batch
options["cellid"] = cellid
"""
ops = loader.split(".")
load_pop_file = ("pop" in ops)
if load_pop_file:
recording_uri = pop_file(siteid=cellid, **options)
elif force_old_loader: # | (batch==316):
log.info("Using 'old' baphy.py loader")
recording_uri, _ = nb.baphy_load_recording_uri(**options)
else:
manager = BAPHYExperiment(batch=batch, cellid=cellid)
recording_uri = manager.get_recording_uri(**options)
return recording_uri
#site = 'ley070a' # good site. A1
#site = 'AMT030a' # low responses, Ok but not as good
#site = 'ley072b' # Primary looking responses with strong contextual effects
modelname = 'resp'
options = {
'batch': 316,
'siteid': site,
'stimfmt': 'envelope',
'rasterfs': 100,
'recache': False,
'runclass': 'CPN',
'stim': False
} #ToDo chace stims, spectrograms???
load_URI = nb.baphy_load_recording_uri(**options)
loaded_rec = recording.load_recording(load_URI)
##########
rec = loaded_rec
dir(rec)
rec.signals
start = 15
stop = 986
signal = rec['resp']
data = signal.rasterize()._data
data.shape
data[2, 1:1000:1]
