def test_recording_loading():
'''
Test the loading and saving of files to various HTTP/S3/File routes.
'''
# NOTE: FOR THIS TEST TO SUCCEED, /auto/data/tmp/recordings/ must not have
# blah.tar.gz or TAR010c-18-1.tar.gz in it.
# Local filesystem
# rec0 = Recording.load("/home/ivar/git/nems/signals/TAR010c-18-1.tar.gz")
rec_path = join(RECORDING_DIR, "eno052d-a1.tgz")
rec0 = Recording.load(rec_path)
rec2 = Recording.load("file://%s" % rec_path)
# HTTP
# rec3 = Recording.load("http://hyrax.ohsu.edu:3000/recordings/eno052d-a1.tgz")
# rec4 = Recording.load("http://hyrax.ohsu.edu:3000/baphy/294/eno052d-a1?stim=0&pupil=0")
# S3
# Direct access (would need AWS CLI lib? Maybe not best idea!)
# TODO: Requires s3 credentials in environment. Probably best if on
# server only; put in nems_db?
#rec5 = Recording.load('s3://mybucket/myfile.tar.gz')
# Indirect access via http:
rec6 = Recording.load("https://s3-us-west-2.amazonaws.com/nemspublic/"
"sample_data/eno052d-a1.tgz")
# Save to a specific tar.gz file
rec0.save('/tmp/tmp.tar.gz')
site = 'TAR010c'
nPCs = 4
batch = 307
fs = 20
rawid = pu.which_rawids(site)
ops = {
'batch': batch,
'pupil': 1,
'rasterfs': fs,
'siteid': site,
'stim': 0,
'rawid': rawid
}
uri = nb.baphy_load_recording_uri(**ops)
rec = Recording.load(uri)
rec['resp'] = rec['resp'].rasterize()
rec = rec.and_mask(['HIT_TRIAL', 'MISS_TRIAL', 'PASSIVE_EXPERIMENT'])
rec = rec.and_mask(['PreStimSilence', 'PostStimSilence'], invert=True)
rec = rec.apply_mask(reset_epochs=True)
# create four state masks
rec = preproc.create_ptd_masks(rec)
# create the appropriate dictionaries of responses
# don't *think* it's so important to balanced reps for
# this analysis since it's parametric (as opposed to the
# discrimination calc which depends greatly on being balanced)
# get list of unique targets present in both a/p. This is for
import random
import numpy as np
from functools import partial
import matplotlib.pyplot as plt
import nems.epoch as ep
import nems.modelspec as ms
import nems.plots.api as nplt
from nems.recording import Recording
# specify directories for loading data and fitted modelspec
signals_dir = '../signals'
#signals_dir = '/home/jacob/auto/data/batch271_fs100_ozgf18/'
modelspecs_dir = '../modelspecs'
# load the data
rec = Recording.load(os.path.join(signals_dir, 'TAR010c-18-1'))
# Add a new signal, respavg, to the recording, in 4 steps
# 1. Fold matrix over all stimuli, returning a dictionary where keys are stimuli
# and each value in the dictionary is (reps X cell X bins)
epochs_to_extract = ep.epoch_names_matching(rec.epochs, '^STIM_')
folded_matrix = rec['resp'].extract_epochs(epochs_to_extract)
# 2. Average over all reps of each stim and save into dict called psth.
per_stim_psth = dict()
for k in folded_matrix.keys():
per_stim_psth[k] = np.nanmean(folded_matrix[k], axis=0)
# 3. Invert the folding to unwrap the psth back out into a predicted spike_dict by
# simply replacing all epochs in the signal with their psth
sites = np.unique([c[:7] for c in cellids])
# loop over sites (to speed loading)
for site in sites:
# load recording(s)
rasterfs = 100
ops = {
'batch': batch,
'siteid': site,
'rasterfs': rasterfs,
'pupil': 1,
'stim': 0,
'recache': False
}
uri = nb.baphy_load_recording_uri(**ops)
rec100 = Recording.load(uri)
rec100['resp'] = rec100['resp'].rasterize()
rec100 = rec100.and_mask(
['HIT_TRIAL', 'CORRECT_REJECT_TRIAL', 'PASSIVE_EXPERIMENT'])
rasterfs = 20
ops = {
'batch': batch,
'siteid': site,
'rasterfs': rasterfs,
'pupil': 1,
'stim': 0,
'recache': False
}
uri = nb.baphy_load_recording_uri(**ops)
rec20 = Recording.load(uri)
# A demonstration of a minimalist model fitting system
from nems.recording import Recording
# ----------------------------------------------------------------------------
# DATA FETCHING
# GOAL: Get your data loaded into memory
# Method #1: Load the data from a local directory
rec = Recording.load('signals/gus027b13_p_PPS/')
# alternative ways to define the data object that could be saved as a
# short(!) string in the dataspec
rec = my_create_recording_fun('signals/gus027b13_p_PPS/')
rec = Recording.load_standard_nems_format('signals/gus027b13_p_PPS/')
# Method #2: Load the data from baphy using the (incomplete, TODO) HTTP API:
# URL = "neuralprediction.org:3003/signals?batch=273&cellid=gus027b13-a1"
# rec = fetch_signals_over_http(URL)
# Method #3: Load the data from S3:
# stimfile="https://s3-us-west-2.amazonaws.com/nemspublic/sample_data/"+cellid+"_NAT_stim_ozgf_c18_fs100.mat"
# respfile="https://s3-us-west-2.amazonaws.com/nemspublic/sample_data/"+cellid+"_NAT_resp_fs100.mat"
# rec = fetch_signals_over_http(stimfile, respfile)
# Method #4: Load the data from a jerb (TODO)
# Method #5: Create a Recording object from a matrix, manually (TODO)
def plot_collapsed_ref_tar(animal, site, cellids=None):
site += "%"
sql = "SELECT DISTINCT cellid, rawid, respfile FROM sCellFile WHERE cellid like %s AND runclassid=%s"
d = nd.pd_query(sql, params=(
site,
42,
))
mfile = []
for f in np.unique(d.respfile):
f_ = f.split('.')[0]
mfile.append('/auto/data/daq/{0}/{1}/{2}'.format(
animal, site[:-2], f_))
if cellids is None:
cellid = np.unique(d.cellid).tolist()
else:
cellid = cellids
options = {
"siteid": site[:-1],
'cellid': cellids,
"mfilename": mfile,
'stim': False,
'pupil': True,
'rasterfs': 1000
}
uri = nb.baphy_load_recording_uri(**options)
rec = Recording.load(uri)
all_pupil = rec['pupil']._data
ncols = len(mfile)
if cellids is None:
cellids = rec['resp'].chans
for c in cellids:
f, ax = plt.subplots(1, ncols, sharey=True, figsize=(12, 5))
ref_base = 0
for i, mf in enumerate(mfile):
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
R = rec.copy()
R['resp'] = R['resp'].rasterize(fs=20)
R['resp'].fs = 20
R = R.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
R = R.and_mask(['HIT_TRIAL']).apply_mask(reset_epochs=True)
resp = R['resp'].extract_channels([c])
tar_reps = resp.extract_epoch('TARGET').shape[0]
tar_m = np.nanmean(resp.extract_epoch('TARGET'),
0).squeeze() * R['resp'].fs
tar_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('TARGET'),
0).squeeze() / np.sqrt(tar_reps)
ref_reps = resp.extract_epoch('REFERENCE').shape[0]
ref_m = np.nanmean(resp.extract_epoch('REFERENCE'),
0).squeeze() * R['resp'].fs
ref_sem = R['resp'].fs * np.nanstd(resp.extract_epoch('REFERENCE'),
0).squeeze() / np.sqrt(ref_reps)
# plot psth's
time = np.linspace(0, len(tar_m) / R['resp'].fs, len(tar_m))
ax[i].plot(time, tar_m, color='red', lw=2)
ax[i].fill_between(time,
tar_m + tar_sem,
tar_m - tar_sem,
color='coral')
time = np.linspace(0, len(ref_m) / R['resp'].fs, len(ref_m))
ax[i].plot(time, ref_m, color='blue', lw=2)
ax[i].fill_between(time,
ref_m + ref_sem,
ref_m - ref_sem,
color='lightblue')
# set title
ax[i].set_title(fn, fontsize=8)
# set labels
ax[i].set_xlabel('Time (s)')
ax[i].set_ylabel('Spk / sec')
# get raster plot baseline
base = np.max(np.concatenate((tar_m + tar_sem, ref_m + ref_sem)))
if base > ref_base:
ref_base = base
for i, mf in enumerate(mfile):
# plot the rasters
fn = mf.split('/')[-1]
ep_mask = [ep for ep in rec.epochs.name if fn in ep]
rast_rec = rec.and_mask(ep_mask).apply_mask(reset_epochs=True)
if '_a_' in fn:
rast_rec = rast_rec.and_mask(['HIT_TRIAL'
]).apply_mask(reset_epochs=True)
rast = rast_rec['resp'].extract_channels([c])
ref_times = np.where(rast.extract_epoch('REFERENCE').squeeze())
base = ref_base
ref_pupil = np.nanmean(
rast_rec['pupil'].extract_epoch('REFERENCE'), -1)
xoffset = rast.extract_epoch(
'TARGET').shape[-1] / rec['resp'].fs + 0.01
ax[i].plot(ref_pupil / np.max(all_pupil) + xoffset,
np.linspace(base, int(base * 2), len(ref_pupil)),
color='k')
ax[i].axvline(xoffset + np.median(all_pupil / np.max(all_pupil)),
linestyle='--',
color='lightgray')
#import pdb; pdb.set_trace()
if ref_times[0].size != 0:
max_rep = ref_pupil.shape[0] - 1
ref_locs = ref_times[0] * (base / max_rep)
ref_locs = ref_locs + base
ax[i].plot(ref_times[1] / rec['resp'].fs,
ref_locs,
'|',
color='blue',
markersize=1)
tar_times = np.where(rast.extract_epoch('TARGET').squeeze())
tar_pupil = np.nanmean(rast_rec['pupil'].extract_epoch('TARGET'),
-1)
tar_base = np.max(ref_locs) + 1
ax[i].plot(tar_pupil / np.max(all_pupil) + xoffset,
np.linspace(tar_base, tar_base + base, len(tar_pupil)),
color='k')
if tar_times[0].size != 0:
max_rep = tar_pupil.shape[0] - 1
tar_locs = tar_times[0] * (base / max_rep)
tar_locs = tar_locs + tar_base
ax[i].plot(tar_times[1] / rec['resp'].fs,
tar_locs,
'|',
color='red',
markersize=1)
# set ylim
#ax[i].set_ylim((0, top))
# set plot aspect
#asp = np.diff(ax[i].get_xlim())[0] / np.diff(ax[i].get_ylim())[0]
#ax[i].set_aspect(asp / 2)
f.suptitle(c, fontsize=8)
f.tight_layout()
def generate_state_corrected_psth(batch=None, modelname=None, cellids=None, siteid=None, movement_mask=False,
gain_only=False, dc_only=False, cache_path=None, recache=False):
"""
Modifies the exisiting recording so that psth signal is the prediction specified
by the modelname. Designed with stategain models in mind. CRH.
If the model doesn't exist already in /auto/users/hellerc/results/, this
will go ahead and fit the model and save it in /auto/users/hellerc/results.
If the fit dir (from xforms) exists, simply reload the result and call this psth.
"""
if siteid is None:
raise ValueError("must specify siteid!")
if cache_path is not None:
fn = cache_path + siteid + '_{}.tgz'.format(modelname.split('.')[1])
if gain_only:
fn = fn.replace('.tgz', '_gonly.tgz')
if 'mvm' in modelname:
fn = fn.replace('.tgz', '_mvm.tgz')
if (os.path.isfile(fn)) & (recache == False):
rec = Recording.load(fn)
return rec
else:
# do the rest of the code
pass
if batch is None or modelname is None:
raise ValueError('Must specify batch and modelname!')
results_table = nd.get_results_file(batch, modelnames=[modelname])
preds = []
ms = []
for cell in cellids:
log.info(cell)
try:
p = results_table[results_table['cellid']==cell]['modelpath'].values[0]
if os.path.isdir(p):
xfspec, ctx = xforms.load_analysis(p)
preds.append(ctx['val'])
ms.append(ctx['modelspec'])
else:
sys.exit('Fit for {0} does not exist'.format(cell))
except:
log.info("WARNING: fit doesn't exist for cell {0}".format(cell))
# Need to add a check to make sure that the preds are the same length (if
# multiple cellids). This could be violated if one cell for example existed
# in a prepassive run but the other didn't and so they were fit differently
file_epochs = []
for pr in preds:
file_epochs += [ep for ep in pr.epochs.name if ep.startswith('FILE')]
unique_files = np.unique(file_epochs)
shared_files = []
for f in unique_files:
if np.sum([1 for file in file_epochs if file == f]) == len(preds):
shared_files.append(str(f))
else:
# this rawid didn't span all cells at the requested site
pass
# mask all file epochs for all preds with the shared file epochs
# and adjust epochs
if (int(batch) == 307) | (int(batch) == 294):
for i, p in enumerate(preds):
preds[i] = p.and_mask(shared_files)
preds[i] = preds[i].apply_mask(reset_epochs=True)
sigs = {}
for i, p in enumerate(preds):
if gain_only:
# update phi
mspec = ms[i]
not_gain_keys = [k for k in mspec[0]['phi'].keys() if '_g' not in k]
for k in not_gain_keys:
mspec[0]['phi'][k] = np.append(mspec[0]['phi'][k][0, 0], np.zeros(mspec[0]['phi'][k].shape[-1]-1))[np.newaxis, :]
pred = mspec.evaluate(p)['pred']
elif dc_only:
mspec = ms[i]
not_dc_keys = [k for k in mspec[0]['phi'].keys() if '_d' not in k]
for k in not_dc_keys:
mspec[0]['phi'][k] = np.append(mspec[0]['phi'][k][0, 0], np.zeros(mspec[0]['phi'][k].shape[-1]-1))[np.newaxis, :]
pred = mspec.evaluate(p)['pred']
else:
pred = p['pred']
if i == 0:
new_psth_sp = p['psth_sp']
new_psth = pred
new_resp = p['resp'].rasterize()
else:
try:
new_psth_sp = new_psth_sp.concatenate_channels([new_psth_sp, p['psth_sp']])
new_psth = new_psth.concatenate_channels([new_psth, pred])
new_resp = new_resp.concatenate_channels([new_resp, p['resp'].rasterize()])
except ValueError:
import pdb; pdb.set_trace()
new_pup = preds[0]['pupil']
sigs['pupil'] = new_pup
if 'pupil_raw' in preds[0].signals.keys():
sigs['pupil_raw'] = preds[0]['pupil_raw']
if 'mask' in preds[0].signals:
new_mask = preds[0]['mask']
sigs['mask'] = new_mask
else:
mask_rec = preds[0].create_mask(True)
new_mask = mask_rec['mask']
sigs['mask'] = new_mask
if 'rem' in preds[0].signals.keys():
rem = preds[0]['rem']
sigs['rem'] = rem
if 'pupil_eyespeed' in preds[0].signals.keys():
new_eyespeed = preds[0]['pupil_eyespeed']
sigs['pupil_eyespeed'] = new_eyespeed
new_psth_sp.name = 'psth_sp'
new_psth.name = 'psth'
new_resp.name = 'resp'
sigs['psth_sp'] = new_psth_sp
sigs['psth'] = new_psth
sigs['resp'] = new_resp
new_rec = Recording(sigs, meta=preds[0].meta)
# make sure mask is cast to bool
new_rec['mask'] = new_rec['mask']._modified_copy(new_rec['mask']._data.astype(bool))
if cache_path is not None:
log.info('caching {}'.format(fn))
new_rec.save_targz(fn)
return new_rec
relative_signals_dir = '../recordings'
relative_modelspecs_dir = '../modelspecs'
# Convert to absolute paths so they can be passed to functions in
# other directories
signals_dir = os.path.abspath(relative_signals_dir)
modelspecs_dir = os.path.abspath(relative_modelspecs_dir)
# ----------------------------------------------------------------------------
# DATA LOADING
# GOAL: Get your data loaded into memory as a Recording object
logging.info('Loading data...')
# Method #1: Load the data from a local directory
rec = Recording.load(os.path.join(signals_dir, 'eno052d-a1.tgz'))
# Method #2: Load the data from baphy using the (incomplete, TODO) HTTP API:
#URL = "http://potoroo:3004/baphy/271/bbl086b-11-1?rasterfs=200"
#rec = Recording.load_url(URL)
logging.info('Generating state signal...')
rec = preproc.make_state_signal(rec, ['pupil'], [''], 'state')
# ----------------------------------------------------------------------------
# INITIALIZE MODELSPEC
# GOAL: Define the model that you wish to test
logging.info('Initializing modelspec(s)...')
relative_signals_dir = '../recordings'
relative_modelspecs_dir = '../modelspecs'
# Convert to absolute paths so they can be passed to functions in
# other directories
signals_dir = os.path.abspath(relative_signals_dir)
modelspecs_dir = os.path.abspath(relative_modelspecs_dir)
# ----------------------------------------------------------------------------
# DATA LOADING
# GOAL: Get your data loaded into memory as a Recording object
logging.info('Loading data...')
# Method #1: Load the data from a local directory
rec = Recording.load(os.path.join(signals_dir, 'TAR010c.NAT.fs50.tgz'))
cellid = 'TAR010c-16-2'
rec['resp'] = rec['resp'].extract_channels([cellid])
# Method #2: Load the data from baphy using the (incomplete, TODO) HTTP API:
#URL = "http://potoroo:3004/baphy/271/bbl086b-11-1?rasterfs=200"
#rec = Recording.load_url(URL)
logging.info('Generating state signal...')
rec = preproc.make_state_signal(rec, ['pupil'], [''], 'state')
# ----------------------------------------------------------------------------
# DATA WITHHOLDING
