def run_decoding_analysis(IsReload=False, **kwargs):
"""
Specialized postprocessor to queue decoding analysis for the model pred data
"""
raise DeprecationWarning("Use 'run_decoding'. It is cleaner.")
if IsReload:
log.info("Reload, skipping rda")
return {}
modelname = kwargs['meta']['modelname']
# figure out movement keywords
threshold = 25
window = 1
ops = modelname.split('-')
for o in ops:
if o.startswith('mvm'):
parms = o.split('.')
for p in parms:
if p.startswith('t'):
threshold = int(p[1:])
elif p.startswith('w'):
window = int(p[1:])
# "base dprime" analysis
mn = f'dprime_mvm-{threshold}-{window}_jk10_zscore_nclvz_fixtdr2-fa'
# noise dims
noise = [-1, 0, 1, 2, 3, 4, 5, 6]
modellist = []
for n in noise:
if n > 0:
modellist.append(mn + f'_noiseDim-{n}')
elif n == -1:
modellist.append(mn + f'_noiseDim-dU')
else:
modellist.append(mn)
# append lv modelname
modellist = [mn + f'_model-LV-{modelname}' for mn in modellist]
script = '/auto/users/hellerc/code/projects/nat_pupil_ms/dprime_new/cache_dprime.py'
python_path = '/auto/users/hellerc/anaconda3/envs/lbhb/bin/python'
nd.enqueue_models(celllist=[kwargs['meta']['cellid'][:7]],
batch=kwargs['meta']['batch'],
modellist=modellist,
executable_path=python_path,
script_path=script,
user='******',
force_rerun=True,
reserve_gb=2)
log.info('Queued decoding analysis')
return {}
def second_fit_pop_models(batch, start_from=None, test_count=None):
all_cellids = nd.get_batch_cells(batch, as_list=True)
if batch == 322:
sites = NAT4_A1_SITES
else:
sites = NAT4_PEG_SITES
cellids = [
c for c in all_cellids
if np.any([c.startswith(s.split('.')[0]) for s in sites])
]
modelnames = []
for k, v in MODELGROUPS.items():
if ('_single' not in k) and ('_exploration' not in k) and (k != 'LN'):
modelnames.extend(v)
iterator = cellids
for siteid in iterator:
for modelname in modelnames[start_from:test_count]:
do_fit = True
if not FORCE_RERUN:
d = nd.pd_query(
"SELECT * FROM Results WHERE cellid like %s and modelname=%s and batch=%s",
params=(siteid + "%", modelname, batch))
if len(d) > 0:
do_fit = False
print(f'Fit exists for {siteid} {batch} {modelname}')
if do_fit:
nd.enqueue_models(
celllist=[siteid],
batch=batch,
modellist=[modelname],
user="******",
#executable_path='/auto/users/jacob/bin/anaconda3/envs/jacob_nems/bin/python',
executable_path=
'/auto/users/svd/bin/miniconda3/envs/tf/bin/python',
script_path=
'/auto/users/jacob/bin/anaconda3/envs/jacob_nems/nems/scripts/fit_single.py'
)
return modelnames
def enqueue_models_view():
"""Call modelfit.enqueue_models with user selections as args."""
user = get_current_user()
# Only pull the numerals from the batch string, leave off the description.
bSelected = request.args.get('bSelected')[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
codeHash = request.args.get('codeHash')
execPath = request.args.get('execPath')
scriptPath = request.args.get('scriptPath')
force_rerun = request.args.get('forceRerun', type=int)
useKamiak = request.args.get('useKamiak', type=int)
kamiakFunction = request.args.get(
'kamiakFunction') # fn to generate scripts
kamiakPath = request.args.get('kamiakPath') # path to store output in
loadKamiak = request.args.get('loadKamiak',
type=int) # check to load results
kamiakResults = request.args.get('kamiakResults') # path to results
useGPU = request.args.get('useGPU', type=int) # path to results
useExacloud = request.args.get('useExacloud', type=int)
exaOHSU = request.args.get('exaOHSU')
exaExec = request.args.get('exaExec')
exaScript = request.args.get('exaScript')
exaLimit = request.args.get('exaLimit')
exaExclude = request.args.get('exaExclude')
exaHighMem = request.args.get('exaHighMem', type=int)
if loadKamiak:
kamiak_to_database(cSelected, bSelected, mSelected, kamiakResults,
execPath, scriptPath)
return jsonify(data=True)
elif useExacloud:
log.info('Starting exacloud jobs!')
enqueue_exacloud_models(cellist=cSelected,
batch=bSelected,
modellist=mSelected,
user=user.username,
linux_user=exaOHSU,
executable_path=exaExec,
script_path=exaScript,
time_limit=exaLimit,
useGPU=useGPU,
high_mem=exaHighMem,
exclude=exaExclude)
return jsonify(data=True)
elif useKamiak:
# kamiakFunction should be a stringified pointer to a function
# that takes a list of cellids, a batch, a list of modelnames,
# and a directory where the output should be stored,
# Ex: kamiakScript = 'nems_lbhb.utils.my_kamiak_function'
try:
kamiak_script = _lookup_fn_at(kamiakFunction, ignore_table=True)
kamiak_script(cSelected, bSelected, mSelected, kamiakPath)
return jsonify(data=True)
except AttributeError:
log.warning('kamiakFunction doesnt exist or is improperly defined')
return jsonify(data=False)
else:
if not codeHash:
codeHash = 'master'
if not execPath:
execPath = None
if not scriptPath:
scriptPath = None
enqueue_models(cSelected,
bSelected,
mSelected,
force_rerun=bool(force_rerun),
user=user.username,
codeHash=codeHash,
executable_path=execPath,
script_path=scriptPath,
GPU_job=useGPU)
return jsonify(data=True)
modellist = np.concatenate([[m+f'_model-LV-{lvstr}' for lvstr in lvmodelnames] for m in modellist]).tolist()
if batch == 331:
modellist = [m.replace('loadpred', loadpredkey) for m in modellist]
if zscore == False:
modellist = [m.replace('_zscore', '') for m in modellist]
if exclude_lowFR:
modellist = [m+f'_rmlowFR-{thresh}' for m in modellist]
if movement_mask is not False:
modellist = [m.replace('dprime_', f'dprime_mvm-{movement_mask[0]}-{movement_mask[1]}_') for m in modellist]
if use_old_cpn:
modellist = [m.replace('dprime_', f'dprime_oldCPN_') for m in modellist]
modellist = [m for m in modellist if '_pr' not in m]
script = '/auto/users/hellerc/code/projects/nat_pupil_ms/dprime_new/cache_dprime.py'
python_path = '/auto/users/hellerc/anaconda3/envs/lbhb/bin/python'
nd.enqueue_models(celllist=sites,
batch=batch,
modellist=modellist,
executable_path=python_path,
script_path=script,
user='******',
force_rerun=force_rerun,
reserve_gb=4)
if exacloud:
from nems_lbhb.exacloud.queue_exacloud_job import enqueue_exacloud_models
#force_rerun=True
lbhb_user = "******"
# exacloud settings:
executable_path = '/home/users/davids/anaconda3/envs/nems/bin/python'
script_path = '/home/users/davids/nems/scripts/fit_single.py'
ssh_key = '/home/svd/.ssh/id_rsa'
user = "******"
enqueue_exacloud_models(cellist=sites,
batch=batch,
modellist=modelnames,
user=lbhb_user,
linux_user=user,
force_rerun=force_rerun,
executable_path=executable_path,
script_path=script_path,
useGPU=True)
else:
script = '/auto/users/hellerc/code/NEMS/scripts/fit_single.py'
python_path = '/auto/users/hellerc/anaconda3/envs/tf/bin/python'
nd.enqueue_models(celllist=sites,
batch=batch,
modellist=modelnames,
executable_path=python_path,
script_path=script,
user='******',
force_rerun=force_rerun,
GPU_job=0)
'psth.fs20.pup-ld-st.pup0.afl0-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.afl.pxf-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.afl.pxf-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.afl0.pxf-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.afl.pxf0-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.afl0.pxf-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.afl0.pxf0-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.afl.pxf0-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.afl0.pxf0-ref-psthfr.s_stategain.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.fil-ref-psthfr.s_sdexp.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.fil-ref-psthfr.s_sdexp.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup.fil0-ref-psthfr.s_sdexp.S_jk.nf20-basic',
'psth.fs20.pup-ld-st.pup0.fil0-ref-psthfr.s_sdexp.S_jk.nf20-basic',
]
batch = 309
force_rerun = True
cells = nd.get_batch_cells(batch).cellid.tolist()
script = '/auto/users/hellerc/code/NEMS/scripts/fit_single.py'
executable = '/auto/users/hellerc/anaconda3/envs/crh_nems/bin/python'
nd.enqueue_models(celllist=cells,
modellist=modellist,
batch=batch,
force_rerun=force_rerun,
script_path=script,
executable_path=executable,
reserve_gb=1,
user='******')
]
modelnames = [
"gtgram.fs100.ch18.mono-ld.pop-norm.l1-sev_wc.Nx50-fir.1x25x50-wc.50xR-lvl.R-dexp.R_tfinit.n.lr1e3.et3.es20-newtf.n.lr1e4.l2:4",
"gtgram.fs100.ch18-ld.pop-norm.l1-sev_wc.Nx50-fir.1x25x50-wc.50xR-lvl.R-dexp.R_tfinit.n.lr1e3.et3.es20-newtf.n.lr1e4.l2:4",
"gtgram.fs100.ch18.bin-ld.pop-norm.l1-sev_wc.Nx50-fir.1x25x50-wc.50xR-lvl.R-dexp.R_tfinit.n.lr1e3.et3.es20-newtf.n.lr1e4.l2:4"
]
force_rerun = False
run_in_lbhb = False
if run_in_lbhb:
# first models, run locally so that recordings get generated.
r = db.enqueue_models(siteids,
batch,
modelnames,
executable_path=executable_path,
script_path=script_path,
GPU_job=GPU_job,
user="******")
for a, b in r:
print(a, b)
else:
# exacloud
# exacloud queue settings:
exa_executable_path = '/home/users/davids/anaconda3/envs/nems/bin/python'
exa_script_path = '/home/users/davids/nems/scripts/fit_single.py'
ssh_key = '/home/svd/.ssh/id_rsa'
user = "******"
lbhb_user = "******"
batch_cells = set(nd.get_batch_cells(batch=batch).cellid)
full_analysis = nd.get_results_file(batch)
already_analyzed = full_analysis.cellid.unique().tolist()
# batch_cells = ['BRT037b-39-1'] # best cell
# iterates over every mode, checks what cells have not been fitted with it and runs the fit command.
for model in modelnames:
ff_model = full_analysis.modelname == model
already_fitted_cells = set(full_analysis.loc[ff_model, 'cellid'])
cells_to_fit = list(batch_cells.difference(already_fitted_cells))
print('model {}, cells to fit:\n{}'.format(model, cells_to_fit))
out = nd.enqueue_models(celllist=cells_to_fit, batch=batch, modellist=[model], user='******', force_rerun=force_rerun,
executable_path=executable_path, script_path=script_path)
for oo in out:
print(oo)
DB_pull = False
if DB_pull is True:
results_table = nd.get_results_file(batch, cellids=list(batch_cells))
preds = []
for cell in batch_cells:
print(cell)
p = results_table[results_table['cellid'] == cell]['modelpath'].values[0]
if os.path.isdir(p):
xfspec, ctx = xforms.load_analysis(p)
