def _queue_fits(batch, modelnames, iterator): for siteid in iterator: for modelname in modelnames: do_fit = True if not FORCE_RERUN: d = nd.pd_query( "SELECT * FROM Results WHERE cellid like %s and modelname=%s and batch=%s", params=(siteid + "%", modelname, batch)) if len(d) > 0: do_fit = False print(f'Fit exists for {siteid} {batch} {modelname}') if do_fit: enqueue_exacloud_models(cellist=[siteid], batch=batch, modellist=[modelname], useGPU=True, **EXACLOUD_SETTINGS)
def enqueue_models_view(): """Call modelfit.enqueue_models with user selections as args.""" user = get_current_user() # Only pull the numerals from the batch string, leave off the description. bSelected = request.args.get('bSelected')[:3] cSelected = request.args.getlist('cSelected[]') mSelected = request.args.getlist('mSelected[]') codeHash = request.args.get('codeHash') execPath = request.args.get('execPath') scriptPath = request.args.get('scriptPath') force_rerun = request.args.get('forceRerun', type=int) useKamiak = request.args.get('useKamiak', type=int) kamiakFunction = request.args.get( 'kamiakFunction') # fn to generate scripts kamiakPath = request.args.get('kamiakPath') # path to store output in loadKamiak = request.args.get('loadKamiak', type=int) # check to load results kamiakResults = request.args.get('kamiakResults') # path to results useGPU = request.args.get('useGPU', type=int) # path to results useExacloud = request.args.get('useExacloud', type=int) exaOHSU = request.args.get('exaOHSU') exaExec = request.args.get('exaExec') exaScript = request.args.get('exaScript') exaLimit = request.args.get('exaLimit') exaExclude = request.args.get('exaExclude') exaHighMem = request.args.get('exaHighMem', type=int) if loadKamiak: kamiak_to_database(cSelected, bSelected, mSelected, kamiakResults, execPath, scriptPath) return jsonify(data=True) elif useExacloud: log.info('Starting exacloud jobs!') enqueue_exacloud_models(cellist=cSelected, batch=bSelected, modellist=mSelected, user=user.username, linux_user=exaOHSU, executable_path=exaExec, script_path=exaScript, time_limit=exaLimit, useGPU=useGPU, high_mem=exaHighMem, exclude=exaExclude) return jsonify(data=True) elif useKamiak: # kamiakFunction should be a stringified pointer to a function # that takes a list of cellids, a batch, a list of modelnames, # and a directory where the output should be stored, # Ex: kamiakScript = 'nems_lbhb.utils.my_kamiak_function' try: kamiak_script = _lookup_fn_at(kamiakFunction, ignore_table=True) kamiak_script(cSelected, bSelected, mSelected, kamiakPath) return jsonify(data=True) except AttributeError: log.warning('kamiakFunction doesnt exist or is improperly defined') return jsonify(data=False) else: if not codeHash: codeHash = 'master' if not execPath: execPath = None if not scriptPath: scriptPath = None enqueue_models(cSelected, bSelected, mSelected, force_rerun=bool(force_rerun), user=user.username, codeHash=codeHash, executable_path=execPath, script_path=scriptPath, GPU_job=useGPU) return jsonify(data=True)
if exacloud: from nems_lbhb.exacloud.queue_exacloud_job import enqueue_exacloud_models #force_rerun=True lbhb_user = "******" # exacloud settings: executable_path = '/home/users/davids/anaconda3/envs/nems/bin/python' script_path = '/home/users/davids/nems/scripts/fit_single.py' ssh_key = '/home/svd/.ssh/id_rsa' user = "******" enqueue_exacloud_models(cellist=sites, batch=batch, modellist=modelnames, user=lbhb_user, linux_user=user, force_rerun=force_rerun, executable_path=executable_path, script_path=script_path, useGPU=True) else: script = '/auto/users/hellerc/code/NEMS/scripts/fit_single.py' python_path = '/auto/users/hellerc/anaconda3/envs/tf/bin/python' nd.enqueue_models(celllist=sites, batch=batch, modellist=modelnames, executable_path=python_path, script_path=script, user='******', force_rerun=force_rerun,
323: modelname_filter.replace('.ver2','')} # ROUND 1, all families pop if 0: modelnames = ALL_FAMILY_POP[:-1] useGPU = True for batch in batches: if useGPU and (batch==322): c = ['NAT4v2'] elif useGPU: c = ['NAT4'] else: c = nd.batch_comp(modelnames=[modelname_filter], batch=batch).index.to_list() enqueue_exacloud_models( cellist=c, batch=batch, modellist=modelnames, user=lbhb_user, linux_user=user, force_rerun=force_rerun, priority=1, executable_path=executable_path_exa, script_path=script_path_exa, useGPU=useGPU) if 0: # dnn single, round 1 modelnames = DNN_SINGLE_MODELS[:5] # ln single, only 1 round modelnames = LN_SINGLE_MODELS[:10] # dnn single, round 2 modelnames = DNN_SINGLE_STAGE2[:5] useGPU = False for batch in batches: cellids = nd.batch_comp(modelnames=[mfb[batch]], batch=batch).index.to_list()