def _queue_fits(batch, modelnames, iterator):
for siteid in iterator:
for modelname in modelnames:
do_fit = True
if not FORCE_RERUN:
d = nd.pd_query(
"SELECT * FROM Results WHERE cellid like %s and modelname=%s and batch=%s",
params=(siteid + "%", modelname, batch))
if len(d) > 0:
do_fit = False
print(f'Fit exists for {siteid} {batch} {modelname}')
if do_fit:
enqueue_exacloud_models(cellist=[siteid],
batch=batch,
modellist=[modelname],
useGPU=True,
**EXACLOUD_SETTINGS)
def enqueue_models_view():
"""Call modelfit.enqueue_models with user selections as args."""
user = get_current_user()
# Only pull the numerals from the batch string, leave off the description.
bSelected = request.args.get('bSelected')[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
codeHash = request.args.get('codeHash')
execPath = request.args.get('execPath')
scriptPath = request.args.get('scriptPath')
force_rerun = request.args.get('forceRerun', type=int)
useKamiak = request.args.get('useKamiak', type=int)
kamiakFunction = request.args.get(
'kamiakFunction') # fn to generate scripts
kamiakPath = request.args.get('kamiakPath') # path to store output in
loadKamiak = request.args.get('loadKamiak',
type=int) # check to load results
kamiakResults = request.args.get('kamiakResults') # path to results
useGPU = request.args.get('useGPU', type=int) # path to results
useExacloud = request.args.get('useExacloud', type=int)
exaOHSU = request.args.get('exaOHSU')
exaExec = request.args.get('exaExec')
exaScript = request.args.get('exaScript')
exaLimit = request.args.get('exaLimit')
exaExclude = request.args.get('exaExclude')
exaHighMem = request.args.get('exaHighMem', type=int)
if loadKamiak:
kamiak_to_database(cSelected, bSelected, mSelected, kamiakResults,
execPath, scriptPath)
return jsonify(data=True)
elif useExacloud:
log.info('Starting exacloud jobs!')
enqueue_exacloud_models(cellist=cSelected,
batch=bSelected,
modellist=mSelected,
user=user.username,
linux_user=exaOHSU,
executable_path=exaExec,
script_path=exaScript,
time_limit=exaLimit,
useGPU=useGPU,
high_mem=exaHighMem,
exclude=exaExclude)
return jsonify(data=True)
elif useKamiak:
# kamiakFunction should be a stringified pointer to a function
# that takes a list of cellids, a batch, a list of modelnames,
# and a directory where the output should be stored,
# Ex: kamiakScript = 'nems_lbhb.utils.my_kamiak_function'
try:
kamiak_script = _lookup_fn_at(kamiakFunction, ignore_table=True)
kamiak_script(cSelected, bSelected, mSelected, kamiakPath)
return jsonify(data=True)
except AttributeError:
log.warning('kamiakFunction doesnt exist or is improperly defined')
return jsonify(data=False)
else:
if not codeHash:
codeHash = 'master'
if not execPath:
execPath = None
if not scriptPath:
scriptPath = None
enqueue_models(cSelected,
bSelected,
mSelected,
force_rerun=bool(force_rerun),
user=user.username,
codeHash=codeHash,
executable_path=execPath,
script_path=scriptPath,
GPU_job=useGPU)
return jsonify(data=True)
if exacloud:
from nems_lbhb.exacloud.queue_exacloud_job import enqueue_exacloud_models
#force_rerun=True
lbhb_user = "******"
# exacloud settings:
executable_path = '/home/users/davids/anaconda3/envs/nems/bin/python'
script_path = '/home/users/davids/nems/scripts/fit_single.py'
ssh_key = '/home/svd/.ssh/id_rsa'
user = "******"
enqueue_exacloud_models(cellist=sites,
batch=batch,
modellist=modelnames,
user=lbhb_user,
linux_user=user,
force_rerun=force_rerun,
executable_path=executable_path,
script_path=script_path,
useGPU=True)
else:
script = '/auto/users/hellerc/code/NEMS/scripts/fit_single.py'
python_path = '/auto/users/hellerc/anaconda3/envs/tf/bin/python'
nd.enqueue_models(celllist=sites,
batch=batch,
modellist=modelnames,
executable_path=python_path,
script_path=script,
user='******',
force_rerun=force_rerun,
323: modelname_filter.replace('.ver2','')}
# ROUND 1, all families pop
if 0:
modelnames = ALL_FAMILY_POP[:-1]
useGPU = True
for batch in batches:
if useGPU and (batch==322):
c = ['NAT4v2']
elif useGPU:
c = ['NAT4']
else:
c = nd.batch_comp(modelnames=[modelname_filter], batch=batch).index.to_list()
enqueue_exacloud_models(
cellist=c, batch=batch, modellist=modelnames,
user=lbhb_user, linux_user=user, force_rerun=force_rerun, priority=1,
executable_path=executable_path_exa, script_path=script_path_exa, useGPU=useGPU)
if 0:
# dnn single, round 1
modelnames = DNN_SINGLE_MODELS[:5]
# ln single, only 1 round
modelnames = LN_SINGLE_MODELS[:10]
# dnn single, round 2
modelnames = DNN_SINGLE_STAGE2[:5]
useGPU = False
for batch in batches:
cellids = nd.batch_comp(modelnames=[mfb[batch]], batch=batch).index.to_list()