def enqueue_models_view():
"""Call modelfit.enqueue_models with user selections as args."""
user = get_current_user()
# Only pull the numerals from the batch string, leave off the description.
bSelected = request.args.get('bSelected')[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
codeHash = request.args.get('codeHash')
execPath = request.args.get('execPath')
scriptPath = request.args.get('scriptPath')
if not codeHash:
codeHash = 'master'
if not execPath:
execPath = None
if not scriptPath:
scriptPath = None
force_rerun = request.args.get('forceRerun', type=int)
enqueue_models(
cSelected,
bSelected,
mSelected,
force_rerun=bool(force_rerun),
user=user.username,
codeHash=codeHash,
executable_path=execPath,
script_path=scriptPath,
)
return jsonify(data=True)
def update_tag_options():
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
tags = [
i[0].split(",") for i in session.query(NarfAnalysis.tags).filter(
or_(
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).distinct().all()
]
# Flatten list of lists into a single list of all tag strings
# and remove leading and trailing whitespace.
taglistbldupspc = [i for sublist in tags for i in sublist]
taglistbldup = [t.strip() for t in taglistbldupspc]
# Reform the list with only unique tags
taglistbl = list(set(taglistbldup))
# Finally, remove any blank tags and sort the list.
taglist = [t for t in taglistbl if t != '']
taglist.sort()
session.close()
return jsonify(taglist=taglist)
def enqueue_models_view():
"""Call modelfit.enqueue_models with user selections as args."""
user = get_current_user()
# Only pull the numerals from the batch string, leave off the description.
bSelected = request.args.get('bSelected')[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
codeHash = request.args.get('codeHash')
jerbQuery = request.args.get('jerbQuery')
log.info('codeHash retrieved properly?: {0}'.format(codeHash))
if not codeHash:
codeHash = 'master'
if not jerbQuery:
jerbQuery = ''
force_rerun = request.args.get('forceRerun', type=int)
enqueue_models(
cSelected,
bSelected,
mSelected,
force_rerun=bool(force_rerun),
user=user.username,
codeHash=codeHash,
jerbQuery=jerbQuery,
)
return jsonify(data=True)
def delete_analysis():
"""Delete the selected analysis from the database."""
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
success = False
aSelected = request.args.get('aSelected')
if len(aSelected) == 0:
return jsonify(success=success)
result = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == aSelected).first())
if result is None:
return jsonify(success=success)
if (result.public or (result.username == user.username)
or (user.labgroup in result.labgroup)):
success = True
session.delete(result)
session.commit()
else:
log.info("You do not have permission to delete this analysis.")
return jsonify(success=success)
session.close()
return jsonify(success=success)
def update_analysis():
"""Update list of analyses after a tag and/or filter selection changes."""
user = get_current_user()
session = Session()
Analysis = Tables()['Analysis']
tagSelected = request.args.getlist('tagSelected[]')
statSelected = request.args.getlist('statSelected[]')
global _previous_update_analysis
previous_tags, previous_stats, previous_analyses, previous_ids = _previous_update_analysis
if (previous_tags == tagSelected) and (previous_stats == statSelected):
analysislist = previous_analyses
analysis_ids = previous_ids
else:
# If special '__any' value is passed, set tag and status to match any
# string in ilike query.
if '__any' in tagSelected:
tagStrings = [Analysis.tags.ilike('%%')]
else:
tagStrings = [
Analysis.tags.ilike('%{0}%'.format(tag))
for tag in tagSelected
]
if '__any' in statSelected:
statStrings = [Analysis.status.ilike('%%')]
else:
statStrings = [
Analysis.status.ilike('%{0}%'.format(stat))
for stat in statSelected
]
analyses = (
session.query(Analysis)
.filter(or_(*tagStrings))
.filter(or_(*statStrings))
.filter(or_(
int(user.sec_lvl) == 9,
Analysis.public == '1',
Analysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
Analysis.username == user.username,
))
.order_by(asc(Analysis.id))
.all()
)
analysislist = [
a.name for a in analyses
]
analysis_ids = [
a.id for a in analyses
]
session.close()
return jsonify(analysislist=analysislist, analysis_ids=analysis_ids)
def site_map():
# only allow users with admin privileges to use this function
user = get_current_user()
if user.sec_lvl < 9:
return Response("Must have admin privileges to view site map")
# get list of defined url routes and their matching function endpoints
links = []
for rule in app.url_map.iter_rules():
if "GET" in rule.methods and has_no_empty_params(rule):
url = url_for(rule.endpoint, **(rule.defaults or {}))
links.append([url, rule.endpoint])
# search through web directory and match function endpoint strings to the
# modules that define those functions
package = nems_web
for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
if ispkg:
subpkg = importlib.import_module("nems_web.{0}".format(modname))
sub_name = modname
for importer, modname, ispkg in pkgutil.iter_modules(
subpkg.__path__):
if "views" not in modname:
continue
mod = importlib.import_module(
"nems_web.{0}.views".format(sub_name))
for link in links:
function_names = [
f[0]
for f in inspect.getmembers(mod, inspect.isfunction)
]
log.info("testing...")
log.info("is: {0} in {1} ?".format(
link[1], function_names))
if link[1] in function_names:
log.info("got past second if statement")
# if matching function is found in module, replace
# endpoint string with module path
path = mod.__file__
# chop off everything before nems parent package
nems_idx = path.find('nems/nems_web')
path = path[nems_idx:]
link[1] = path
log.info("Defined routes:\n")
html = "<h3> Defined routes: </h3>"
for link in links:
log.info("url route for: {0} \n goes to endpoint: {1}".format(
link[0], link[1]))
html += ("<br><p> url: {0} </p><p> is defined in: {1}</p>".format(
link[0], link[1]))
return Response(html)
def get_saved_selections():
session = Session()
user = get_current_user()
user_entry = (session.query(NarfUsers).filter(
NarfUsers.username == user.username).first())
if not user_entry:
return jsonify(response="user not logged in, can't load selections")
selections = user_entry.selections
null = False
if not selections:
null = True
session.close()
return jsonify(selections=selections, null=null)
def set_saved_selections():
user = get_current_user()
if not user.username:
return jsonify(
response="user not logged in, can't save selections",
null=True,
)
session = Session()
saved_selections = request.args.get('stringed_selections')
user_entry = (session.query(NarfUsers).filter(
NarfUsers.username == user.username).first())
user_entry.selections = saved_selections
session.commit()
session.close()
return jsonify(response='selections saved', null=False)
def update_status_options():
user = get_current_user()
session = Session()
statuslist = [
i[0] for i in session.query(NarfAnalysis.status).filter(
or_(
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).distinct().all()
]
session.close()
return jsonify(statuslist=statuslist)
def enqueue_models_view():
"""Call modelfit.enqueue_models with user selections as args."""
user = get_current_user()
# Only pull the numerals from the batch string, leave off the description.
bSelected = request.args.get('bSelected')[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
codeHash = request.args.get('codeHash')
execPath = request.args.get('execPath')
scriptPath = request.args.get('scriptPath')
force_rerun = request.args.get('forceRerun', type=int)
useKamiak = request.args.get('useKamiak', type=int)
kamiakFunction = request.args.get(
'kamiakFunction') # fn to generate scripts
kamiakPath = request.args.get('kamiakPath') # path to store output in
loadKamiak = request.args.get('loadKamiak',
type=int) # check to load results
kamiakResults = request.args.get('kamiakResults') # path to results
useGPU = request.args.get('useGPU', type=int) # path to results
useExacloud = request.args.get('useExacloud', type=int)
exaOHSU = request.args.get('exaOHSU')
exaExec = request.args.get('exaExec')
exaScript = request.args.get('exaScript')
exaLimit = request.args.get('exaLimit')
exaExclude = request.args.get('exaExclude')
exaHighMem = request.args.get('exaHighMem', type=int)
if loadKamiak:
kamiak_to_database(cSelected, bSelected, mSelected, kamiakResults,
execPath, scriptPath)
return jsonify(data=True)
elif useExacloud:
log.info('Starting exacloud jobs!')
enqueue_exacloud_models(cellist=cSelected,
batch=bSelected,
modellist=mSelected,
user=user.username,
linux_user=exaOHSU,
executable_path=exaExec,
script_path=exaScript,
time_limit=exaLimit,
useGPU=useGPU,
high_mem=exaHighMem,
exclude=exaExclude)
return jsonify(data=True)
elif useKamiak:
# kamiakFunction should be a stringified pointer to a function
# that takes a list of cellids, a batch, a list of modelnames,
# and a directory where the output should be stored,
# Ex: kamiakScript = 'nems_lbhb.utils.my_kamiak_function'
try:
kamiak_script = _lookup_fn_at(kamiakFunction, ignore_table=True)
kamiak_script(cSelected, bSelected, mSelected, kamiakPath)
return jsonify(data=True)
except AttributeError:
log.warning('kamiakFunction doesnt exist or is improperly defined')
return jsonify(data=False)
else:
if not codeHash:
codeHash = 'master'
if not execPath:
execPath = None
if not scriptPath:
scriptPath = None
enqueue_models(cSelected,
bSelected,
mSelected,
force_rerun=bool(force_rerun),
user=user.username,
codeHash=codeHash,
executable_path=execPath,
script_path=scriptPath,
GPU_job=useGPU)
return jsonify(data=True)
def main_view():
"""Initialize the nems_analysis landing page.
Queries the database to get lists of available analyses, batches,
status filters, tag filters, and results columns.
Specifies defaults for results columns, row limit and sort column.
Returns:
--------
main.html : template
The landing page template rendered with variables for analysislist,
batchlist, collist, defaultcols, measurelist, defaultrowlimit,
sortlist, defaultsort, statuslist, and taglist.
"""
# TODO: figure out how to integrate sec_lvl/superuser mode
# maybe need to add sec_lvl column to analysis/batches/results?
# then can compare in query ex: if user.sec_lvl > analysis.sec_lvl
user = get_current_user()
session = Session()
db_tables = Tables()
NarfResults = db_tables['NarfResults']
NarfAnalysis = db_tables['NarfAnalysis']
NarfBatches = db_tables['NarfBatches']
sBatch = db_tables['sBatch']
# .all() returns a list of tuples, so it's necessary to pull the
# name elements out into a list by themselves.
analyses = (session.query(NarfAnalysis).filter(
or_(
int(user.sec_lvl) == 9,
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).order_by(asc(NarfAnalysis.id)).all())
analysislist = [a.name for a in analyses]
analysis_ids = [a.id for a in analyses]
batchids = [
i[0] for i in session.query(NarfBatches.batch).distinct()
#.filter(or_(
# int(user.sec_lvl) == 9,
# NarfBatches.public == '1',
# NarfBatches.labgroup.ilike('%{0}%'.format(user.labgroup)),
# NarfBatches.username == user.username,
# ))
.all()
]
batchnames = []
for i in batchids:
name = (session.query(sBatch.name).filter(sBatch.id == i).first())
if not name:
batchnames.append('')
else:
batchnames.append(name.name)
batchlist = [(batch + ': ' + batchnames[i])
for i, batch in enumerate(batchids)]
batchlist.sort()
# Default settings for results display.
# TODO: let user choose their defaults and save for later sessions
# cols are in addition to cellid, modelname and batch,
# which are set up to be required
defaultcols = n_ui.cols
defaultrowlimit = n_ui.rowlimit
defaultsort = n_ui.sort
measurelist = n_ui.measurelist
statuslist = [
i[0] for i in session.query(NarfAnalysis.status).filter(
NarfAnalysis.name.in_(analysislist)).distinct().all()
]
# Separate tags into list of lists of strings.
tags = [
i[0].split(",") for i in session.query(NarfAnalysis.tags).filter(
NarfAnalysis.name.in_(analysislist)).distinct().all()
]
# Flatten list of lists into a single list of all tag strings
# and remove leading and trailing whitespace.
taglistbldupspc = [i for sublist in tags for i in sublist]
taglistbldup = [t.strip() for t in taglistbldupspc]
# Reform the list with only unique tags
taglistbl = list(set(taglistbldup))
# Finally, remove any blank tags and sort the list.
taglist = [t for t in taglistbl if t != '']
taglist.sort()
# Returns all columns in the format 'NarfResults.columnName,'
# then removes the leading 'NarfResults.' from each string
collist = ['%s' % (s) for s in NarfResults.__table__.columns]
collist = [s.replace('NarfResults.', '') for s in collist]
sortlist = copy.deepcopy(collist)
# Remove cellid and modelname from options toggles- make them required.
required_cols = n_ui.required_cols
for col in required_cols:
collist.remove(col)
# imported at top from PlotGenerator
plotTypeList = PLOT_TYPES
# imported at top from nems_web.run_scrits.script_utils
scriptList = scan_for_scripts()
session.close()
return render_template('main.html',
analysislist=analysislist,
analysis_ids=analysis_ids,
batchlist=batchlist,
collist=collist,
defaultcols=defaultcols,
measurelist=measurelist,
defaultrowlimit=defaultrowlimit,
sortlist=sortlist,
defaultsort=defaultsort,
statuslist=statuslist,
taglist=taglist,
plotTypeList=plotTypeList,
username=user.username,
seclvl=int(user.sec_lvl),
iso=n_ui.iso,
snr=n_ui.snr,
snri=n_ui.snri,
scripts=scriptList,
bokeh_version=bokeh_version)
def edit_analysis():
"""Take input from Analysis Editor modal and save it to the database.
Button : Edit Analysis
"""
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
modTime = datetime.datetime.now().replace(microsecond=0)
eName = request.args.get('name')
eId = request.args.get('id')
eStatus = request.args.get('status')
eTags = request.args.get('tags')
eQuestion = request.args.get('question')
eAnswer = request.args.get('answer')
eLoad = request.args.get('load')
eMod = request.args.get('mod')
eFit = request.args.get('fit')
eTree = json.dumps([eLoad, eMod, eFit])
if eId == '__none':
checkExists = False
else:
checkExists = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == eId).first())
if checkExists:
a = checkExists
if (a.public or (user.labgroup in a.labgroup)
or (a.username == user.username)):
a.name = eName
a.status = eStatus
a.question = eQuestion
a.answer = eAnswer
a.tags = eTags
try:
a.lastmod = modTime
except:
a.lastmod = str(modTime)
a.modeltree = eTree
else:
log.info("You do not have permission to modify this analysis.")
return jsonify(success=("failed"))
# If it doesn't exist, add new sql alchemy object with the
# appropriate attributes, which should get assigned to a new id
else:
# TODO: Currently copies user's labgroup by default.
# Is that the behavior we want?
try:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=modTime,
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
except:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=str(modTime),
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
session.add(a)
addedName = a.name
session.commit()
session.close()
# After handling submissions, return user to main page so that it
# refreshes with new analysis included in list
return jsonify(success="Analysis %s saved successfully." % addedName)
def update_results():
"""Update the results table after a batch, cell or model selection
is changed.
"""
user = get_current_user()
session = Session()
NarfResults = Tables()['NarfResults']
nullselection = """
MUST SELECT A BATCH AND ONE OR MORE CELLS AND
ONE OR MORE MODELS BEFORE RESULTS WILL UPDATE
"""
bSelected = request.args.get('bSelected')
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
colSelected = request.args.getlist('colSelected[]')
# If no batch, cell or model is selected, display an error message.
if (len(bSelected) == 0) or (not cSelected) or (not mSelected):
return jsonify(resultstable=nullselection)
# Only get numerals for selected batch.
bSelected = bSelected[:3]
# Use default value of 500 if no row limit is specified.
rowlimit = request.args.get('rowLimit', 500)
ordSelected = request.args.get('ordSelected')
# Parse string into appropriate sqlalchemy method
if ordSelected == 'asc':
ordSelected = asc
elif ordSelected == 'desc':
ordSelected = desc
sortSelected = request.args.get('sortSelected', 'cellid')
# Always add cellid and modelname to column lists,
# since they are required for selection behavior.
cols = [
getattr(NarfResults, 'cellid'),
getattr(NarfResults, 'modelname'),
]
cols += [
getattr(NarfResults, c) for c in colSelected
if hasattr(NarfResults, c)
]
# Package query results into a DataFrame
results = psql.read_sql_query(
Query(cols, session).filter(NarfResults.batch == bSelected).filter(
NarfResults.cellid.in_(cSelected)).filter(
NarfResults.modelname.in_(mSelected)).filter(
or_(
int(user.sec_lvl) == 9,
NarfResults.public == '1',
NarfResults.labgroup.ilike('%{0}%'.format(
user.labgroup)),
NarfResults.username == user.username,
)).order_by(ordSelected(getattr(
NarfResults, sortSelected))).limit(rowlimit).statement,
session.bind)
with pd.option_context('display.max_colwidth', -1):
resultstable = results.to_html(
index=False,
classes="table-hover table-condensed",
)
session.close()
return jsonify(resultstable=resultstable)
def make_jerb_json():
user = get_current_user()
session = Session()
# .all() returns a list of tuples, so it's necessary to pull the
# name elements out into a list by themselves.
analyses = (session.query(NarfAnalysis).filter(
or_(
int(user.sec_lvl) == 9,
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).order_by(asc(NarfAnalysis.id)).all())
analysislist = [a.name for a in analyses]
batchids = [
i[0] for i in session.query(NarfBatches.batch).distinct()
#.filter(or_(
# int(user.sec_lvl) == 9,
# NarfBatches.public == '1',
# NarfBatches.labgroup.ilike('%{0}%'.format(user.labgroup)),
# NarfBatches.username == user.username,
# ))
.all()
]
batchnames = []
for i in batchids:
name = (session.query(sBatch.name).filter(sBatch.id == i).first())
if not name:
batchnames.append('')
else:
batchnames.append(name.name)
batchlist = [(batch + ': ' + batchnames[i])
for i, batch in enumerate(batchids)]
batchlist.sort()
session.close()
s3 = boto3.resource('s3')
bucket = s3.Bucket('nemsdata')
jerb_json = {
'name': 'Analysis',
'children': [],
}
for i, analysis in enumerate(analysislist):
jerb_json['children'].append({'name': analysis, 'children': []})
jerb_json['children'][i]['children'].extend([{
'name':
'batch',
'children': [{
'name': batch,
'leaf': 1
} for batch in batchlist]
}, {
'name':
'models',
'children': [{
'name': model,
'leaf': 1
} for model in ['fake', 'model', 'list']]
}, {
'name':
'data',
'children': [{
'name': obj.key.strip('nems_in_cache/batch291/'),
'leaf': 1
} for i, obj in enumerate(
bucket.objects.filter(Prefix='nems_in_cache/batch291/'))
if i < 20]
}])
return jerb_json
def edit_analysis():
"""Take input from Analysis Editor modal and save it to the database.
Button : Edit Analysis
"""
user = get_current_user()
session = Session()
modTime = datetime.datetime.now().replace(microsecond=0)
eName = request.args.get('name')
eId = request.args.get('id')
eStatus = request.args.get('status')
eTags = request.args.get('tags')
eQuestion = request.args.get('question')
eAnswer = request.args.get('answer')
eTree = request.args.get('tree')
#TODO: add checks to require input inside form fields
# or allow blank so that people can erase stuff?
# Turned this off for now -- can re-enable when rule needs are more stable
# Make sure the keyword combination is valid using nems.keyword_rules
#try:
# mf = ModelFinder(eTree)
# for modelname in mf.modellist:
# keyword_test_routine(modelname)
#except Exception as e:
# return jsonify(success='Analysis not saved: \n' + str(e))
if eId == '__none':
checkExists = False
else:
checkExists = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == eId).first())
if checkExists:
a = checkExists
if (a.public or (user.labgroup in a.labgroup)
or (a.username == user.username)):
a.name = eName
a.status = eStatus
a.question = eQuestion
a.answer = eAnswer
a.tags = eTags
try:
a.lastmod = modTime
except:
a.lastmod = str(modTime)
a.modeltree = eTree
else:
log.info("You do not have permission to modify this analysis.")
return jsonify(success=("failed"))
# If it doesn't exist, add new sql alchemy object with the
# appropriate attributes, which should get assigned to a new id
else:
# TODO: Currently copies user's labgroup by default.
# Is that the behavior we want?
try:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=modTime,
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
except:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=str(modTime),
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
session.add(a)
# For verifying correct logging - comment these out
# when not needed for testing.
#log.info("Added the following analysis to database:")
#log.info("------------------")
#log.info("name:"); log.info(a.name)
#log.info("question:"); log.info(a.question)
#log.info("answer:"); log.info(a.answer)
#log.info("status:"); log.info(a.status)
#log.info("tags:"); log.info(a.tags)
#log.info("model tree:"); log.info(a.modeltree)
#log.info("-----------------\n\n")
addedName = a.name
session.commit()
session.close()
# After handling submissions, return user to main page so that it
# refreshes with new analysis included in list
return jsonify(success="Analysis %s saved successfully." % addedName)
