def get_current_analysis():
"""Populate the Analysis Editor form with the database contents for the
currently selected analysis.
"""
session = Session()
aSelected = request.args.get('aSelected')
# If no analysis was selected, fill fields with blank text to
# mimic 'New Analysis' behavior.
if len(aSelected) == 0:
return jsonify(
name='',
status='',
tags='',
question='',
answer='',
tree='',
)
a = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == aSelected).first())
session.close()
return jsonify(
id=a.id,
name=a.name,
status=a.status,
tags=a.tags,
question=a.question,
answer=a.answer,
tree=a.modeltree,
)
def load_user(user_id):
session = Session()
NarfUsers = Tables()['NarfUsers']
try:
# get email match from user database
# (needs to be stored as unicode per flask-login)
sqla_user = (session.query(NarfUsers).filter(
NarfUsers.email == user_id).first())
if not sqla_user:
raise Exception()
# assign attrs from table object to active user instance
user = User(
username=sqla_user.username,
# password should be stored as bcrypt hash
# (generated when registered)
password=sqla_user.password,
labgroup=sqla_user.labgroup,
sec_lvl=sqla_user.sec_lvl,
)
return user
except Exception as e:
log.exception(e)
return None
finally:
session.close()
def cell_details(cellid):
# Just redirects to celldb penetration info for now
# Keeping as separate function incase want to add extra stuff later
# (as opposed to just putting link directly in javascript)
session = Session()
url_root = 'http://hyrax.ohsu.edu/celldb/peninfo.php?penid='
i = cellid.find('-')
cellid = cellid[:i]
try:
result = (session.query(gCellMaster).filter(
gCellMaster.cellid == cellid).first())
except Exception as e:
print(e)
return Response("Couldn't open cell file -- "
"this option is disabled for non-LBHB setup.")
if not result:
# return an error response instead?
# or make it a dud link? seems wasteful to refresh page
# could also redirect to celldb home page
return redirect(url_for('main_view'))
penid = result.penid
session.close()
return redirect(url_root + str(penid))
def update_tag_options():
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
tags = [
i[0].split(",") for i in session.query(NarfAnalysis.tags).filter(
or_(
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).distinct().all()
]
# Flatten list of lists into a single list of all tag strings
# and remove leading and trailing whitespace.
taglistbldupspc = [i for sublist in tags for i in sublist]
taglistbldup = [t.strip() for t in taglistbldupspc]
# Reform the list with only unique tags
taglistbl = list(set(taglistbldup))
# Finally, remove any blank tags and sort the list.
taglist = [t for t in taglistbl if t != '']
taglist.sort()
session.close()
return jsonify(taglist=taglist)
def check_analysis_exists():
"""Check for a duplicate analysis name when an Analysis Editor form is
submitted. If a duplicate exists, warn the user before overwriting.
"""
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
nameEntered = request.args.get('nameEntered')
analysisId = request.args.get('analysisId')
exists = False
result = (session.query(NarfAnalysis).filter(
NarfAnalysis.name == nameEntered).first())
# only set to True if id is different, so that
# overwriting own analysis doesn't cause flag
if result and (analysisId == '__none' or
(int(result.id) != int(analysisId))):
exists = True
session.close()
return jsonify(exists=exists)
def get_id(self):
# user_id should be the unicode rep of e-mail address
# (should be stored in db table in that format)
session = Session()
user_id = (session.query(NarfUsers.email).filter(
NarfUsers.username == self.username).first())
session.close()
return user_id
def update_cells():
"""Update the list of cells in the cell selector after a batch
is selected (this will cascade from an analysis selection).
Also updates current batch in NarfAnalysis for current analysis.
"""
session = Session()
# Only get the numerals for the selected batch, not the description.
bSelected = request.args.get('bSelected')
aSelected = request.args.get('aSelected')
celllist = [
i[0] for i in session.query(NarfBatches.cellid).filter(
NarfBatches.batch == bSelected[:3]).all()
]
batchname = (session.query(sBatch).filter(
sBatch.id == bSelected[:3]).first())
if batchname:
batch = str(bSelected[:3] + ': ' + batchname.name)
else:
batch = bSelected
analysis = (session.query(NarfAnalysis).filter(
NarfAnalysis.name == aSelected).first())
# don't change batch association if batch is blank
if analysis and bSelected:
analysis.batch = batch
session.commit()
session.close()
return jsonify(celllist=celllist)
def delete_analysis():
"""Delete the selected analysis from the database."""
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
success = False
aSelected = request.args.get('aSelected')
if len(aSelected) == 0:
return jsonify(success=success)
result = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == aSelected).first())
if result is None:
return jsonify(success=success)
if (result.public or (result.username == user.username)
or (user.labgroup in result.labgroup)):
success = True
session.delete(result)
session.commit()
else:
log.info("You do not have permission to delete this analysis.")
return jsonify(success=success)
session.close()
return jsonify(success=success)
def get_saved_selections():
session = Session()
user = get_current_user()
user_entry = (session.query(NarfUsers).filter(
NarfUsers.username == user.username).first())
if not user_entry:
return jsonify(response="user not logged in, can't load selections")
selections = user_entry.selections
null = False
if not selections:
null = True
session.close()
return jsonify(selections=selections, null=null)
def register():
errors = ''
form = RegistrationForm(request.form)
if request.method == 'POST':
if form.validate():
session = Session()
NarfUsers = Tables()['NarfUsers']
new_user = NarfUsers(
username=form.username.data,
password=bcrypt.generate_password_hash(form.password.data),
email=(form.email.data).encode('utf-8'),
firstname=form.firstname.data,
lastname=form.lastname.data,
)
session.add(new_user)
session.commit()
session.close()
return redirect(url_for('login'))
else:
log.info('form errors:')
log.info(form.errors)
return render_template(
'account_management/register.html',
form=form,
errors=form.errors,
)
return render_template(
'account_management/register.html',
form=form,
errors=errors,
)
def update_status_options():
user = get_current_user()
session = Session()
statuslist = [
i[0] for i in session.query(NarfAnalysis.status).filter(
or_(
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).distinct().all()
]
session.close()
return jsonify(statuslist=statuslist)
def update_analysis_details():
"""Update contents of the analysis details popover when the analysis
selection is changed.
"""
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
# TODO: Find a better/centralized place to store these options.
# Columns to display in detail popup - add/subtract here if desired.
detailcols = n_ui.detailcols
aSelected = request.args.get('aSelected')
cols = [
getattr(NarfAnalysis, c) for c in detailcols
if hasattr(NarfAnalysis, c)
]
# Package query results into a DataFrame
results = psql.read_sql_query(
Query(cols, session).filter(NarfAnalysis.name == aSelected).statement,
session.bind)
detailsHTML = """"""
if results.size > 0:
for col in detailcols:
# Use a single line for id and status columns
if (col == 'id') or (col == 'status'):
detailsHTML += """
<p><strong>%s</strong>: %s</p>
""" % (col, results[col].iat[0])
# Use a header + paragraph for everything else
else:
detailsHTML += """
<h5><strong>%s</strong>:</h5>
<p>%s</p>
""" % (col, results[col].iat[0])
session.close()
return jsonify(details=detailsHTML)
def update_batch():
"""Update current batch selection after an analysis is selected."""
session = Session()
blank = 0
aSelected = request.args.get('aSelected', type=str)
batch = (session.query(
NarfAnalysis.batch).filter(NarfAnalysis.name == aSelected).first())
try:
batch = batch.batch
except Exception as e:
log.info(e)
batch = ''
blank = 1
session.close()
return jsonify(batch=batch, blank=blank)
def __call__(self, form, field):
session = Session()
exists = (
session.query(NarfUsers)
.filter(NarfUsers.email == field.data)
.first()
)
if exists:
session.close()
raise ValidationError(self.message)
session.close()
def update_analysis():
"""Update list of analyses after a tag and/or filter selection changes."""
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
tagSelected = request.args.getlist('tagSelected[]')
statSelected = request.args.getlist('statSelected[]')
# If special '__any' value is passed, set tag and status to match any
# string in ilike query.
if '__any' in tagSelected:
tagStrings = [NarfAnalysis.tags.ilike('%%')]
else:
tagStrings = [
NarfAnalysis.tags.ilike('%{0}%'.format(tag)) for tag in tagSelected
]
if '__any' in statSelected:
statStrings = [NarfAnalysis.status.ilike('%%')]
else:
statStrings = [
NarfAnalysis.status.ilike('%{0}%'.format(stat))
for stat in statSelected
]
analyses = (session.query(NarfAnalysis).filter(or_(*tagStrings)).filter(
or_(*statStrings)).filter(
or_(
int(user.sec_lvl) == 9,
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).order_by(asc(NarfAnalysis.id)).all())
analysislist = [a.name for a in analyses]
analysis_ids = [a.id for a in analyses]
session.close()
return jsonify(analysislist=analysislist, analysis_ids=analysis_ids)
def __call__(self, form, field):
session = Session()
NarfUsers = Tables()['NarfUsers']
exists = (
session.query(NarfUsers)
.filter(NarfUsers.username == field.data)
.first()
)
if exists:
session.close()
raise ValidationError(self.message)
session.close()
def update_models():
"""Update the list of modelnames in the model selector after an
analysis is selected.
"""
session = Session()
aSelected = request.args.get('aSelected', type=str)
modeltree = (session.query(
NarfAnalysis.modeltree).filter(NarfAnalysis.name == aSelected).first())
# Pass modeltree string from NarfAnalysis to a ModelFinder constructor,
# which will use a series of internal methods to convert the tree string
# to a list of model names.
if modeltree:
mf = ModelFinder(modeltree[0])
else:
return jsonify(modellist="Model tree not found.")
session.close()
return jsonify(modellist=mf.modellist)
def batch_performance():
session = Session()
cSelected = request.form['cSelected']
bSelected = request.form['bSelected'][:3]
mSelected = request.form['mSelected']
findAll = request.form['findAll']
cSelected = cSelected.split(',')
mSelected = mSelected.split(',')
if int(findAll):
results = psql.read_sql_query(
session.query(
NarfResults.cellid, NarfResults.modelname,
NarfResults.r_test
)
.filter(NarfResults.batch == bSelected)
.statement,
session.bind
)
else:
results = psql.read_sql_query(
session.query(
NarfResults.cellid, NarfResults.modelname,
NarfResults.r_test
)
.filter(NarfResults.batch == bSelected)
.filter(NarfResults.cellid.in_(cSelected))
.filter(NarfResults.modelname.in_(mSelected))
.statement,
session.bind
)
# get back list of models that matched other query criteria
results_models = [
m for m in
list(set(results['modelname'].values.tolist()))
]
# filter mSelected to match results models so that list is in the
# same order as on web UI
ordered_models = [
m for m in mSelected
if m in results_models
]
report = Performance_Report(results, bSelected, ordered_models)
report.generate_plot()
session.close()
return render_template(
'batch_performance.html', script=report.script, div=report.div,
bokeh_version=bokeh_version,
)
def set_saved_selections():
user = get_current_user()
if not user.username:
return jsonify(
response="user not logged in, can't save selections",
null=True,
)
session = Session()
saved_selections = request.args.get('stringed_selections')
user_entry = (session.query(NarfUsers).filter(
NarfUsers.username == user.username).first())
user_entry.selections = saved_selections
session.commit()
session.close()
return jsonify(response='selections saved', null=False)
def get_preview():
"""Queries the database for the filepath to the preview image
for the selected cell, batch and model combination(s)
"""
session = Session()
NarfResults = Tables()['NarfResults']
# Only get the numerals for the selected batch, not the description.
bSelected = request.args.get('bSelected', type=str)[:3]
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
figurefile = None
path = (session.query(NarfResults).filter(
NarfResults.batch == bSelected).filter(
NarfResults.cellid.in_(cSelected)).filter(
NarfResults.modelname.in_(mSelected)).first())
if not path:
session.close()
return jsonify(image='missing preview')
else:
figurefile = str(path.figurefile)
session.close()
# Another temporary compatibility hack to convert
# s3://... to https://
if figurefile.startswith('s3'):
prefix = 'https://s3-us-west2.amazonaws.com'
parsed = urlparse(figurefile)
bucket = parsed.netloc
path = parsed.path
figurefile = prefix + '/' + bucket + '/' + path
# TODO: this should eventually be the only thing that gets
# called - above try/except ugliness is temporary for
# backwards compatibility
image_bytes = load_resource(figurefile)
b64img = str(b64encode(image_bytes))[2:-1]
return jsonify(image=b64img)
def update_models():
"""Update the list of modelnames in the model selector after an
analysis is selected.
"""
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
aSelected = request.args.get('aSelected', type=str)
extraModels = request.args.get('extraModels', type=str)
extraAnalyses = request.args.get('extraAnalyses', type=str)
extraModels = [s.strip() for s in extraModels.split(',')]
extraAnalyses = [s.strip() for s in extraAnalyses.split(',')]
modeltree = (session.query(
NarfAnalysis.modeltree).filter(NarfAnalysis.name == aSelected).first())
# Pass modeltree string from NarfAnalysis to a ModelFinder constructor,
# which will use a series of internal methods to convert the tree string
# to a list of model names.
# Then add any additional models specified in extraModels, and add
# model_lists from extraAnalyses.
if modeltree and modeltree[0]:
model_list = _get_models(modeltree[0])
extraModels = [
m for m in extraModels if (m not in model_list and m.strip() != '')
]
model_list.extend(extraModels)
if extraAnalyses:
analyses = (session.query(NarfAnalysis.modeltree).filter(
NarfAnalysis.name.in_(extraAnalyses)).all())
for t in [a.modeltree for a in analyses]:
extras = [m for m in _get_models(t) if m not in model_list]
model_list.extend(extras)
else:
return jsonify(modellist="Model tree not found.")
session.close()
return jsonify(modellist=model_list)
def fit_report():
session = Session()
db_tables = Tables()
tQueue = db_tables['tQueue']
NarfResults = db_tables['NarfResults']
cSelected = request.args.getlist('cSelected[]')
bSelected = request.args.get('bSelected')[:3]
mSelected = request.args.getlist('mSelected[]')
multi_index = pd.MultiIndex.from_product([mSelected, cSelected],
names=['modelname', 'cellid'])
status = pd.DataFrame(index=multi_index, columns=['yn'])
tuples = list(itertools.product(cSelected, [bSelected], mSelected))
notes = ['{0}/{1}/{2}'.format(t[0], t[1], t[2]) for t in tuples]
qdata = psql.read_sql_query(
session.query(tQueue).filter(tQueue.note.in_(notes)).statement,
session.bind,
)
results = psql.read_sql_query(
session.query(
NarfResults.cellid,
NarfResults.batch,
NarfResults.modelname,
).filter(NarfResults.batch == bSelected).filter(
NarfResults.cellid.in_(cSelected)).filter(
NarfResults.modelname.in_(mSelected)).statement, session.bind)
for i, t in enumerate(tuples):
yn = 0.3 # missing
try:
complete = qdata.loc[qdata['note'] == notes[i], 'complete'].iloc[0]
if complete < 0:
yn = 0.4 # in progress
elif complete == 0:
yn = 0.5 # not started
elif complete == 1:
yn = 0.6 # finished
elif complete == 2:
yn = 0 # dead entry
else:
pass # unknown value, so leave as missing?
except:
try:
result = results.loc[(results['cellid'] == t[0])
& (results['batch'] == int(t[1]))
& (results['modelname'] == t[2]),
'cellid'].iloc[0]
yn = 0.6
except:
pass
status['yn'].loc[t[2], t[0]] = yn
status.reset_index(inplace=True)
status = status.pivot(index='cellid', columns='modelname', values='yn')
status = status[status.columns].astype(float)
report = Fit_Report(status)
report.generate_plot()
session.close()
image = str(b64encode(report.img_str))[2:-1]
return jsonify(image=image)
def form_data_array(
session,
batch,
cells,
models,
columns=None,
only_fair=True,
include_outliers=False,
):
# TODO: figure out a good way to form this from the existing
# dataframe instead of making a new one then copying over.
# Should be able to just re-index then apply some
# lambda function over vectorized dataframe for filtering?
session = Session()
NarfResults = Tables()['NarfResults']
data = psql.read_sql_query(
session.query(NarfResults).filter(NarfResults.batch == batch).filter(
NarfResults.cellid.in_(cells)).filter(
NarfResults.modelname.in_(models)).statement, session.bind)
if not columns:
columns = data.columns.values.tolist()
multiIndex = pd.MultiIndex.from_product(
[cells, models],
names=['cellid', 'modelname'],
)
newData = pd.DataFrame(
index=multiIndex,
columns=columns,
)
newData.sort_index()
# columns that don't contain performance data - this will be excluded
# from outlier checks.
non_comp_columns = [
'id',
'cellid',
'modelname',
'batch',
'n_parms',
'figurefile',
'githash',
'lastmod',
'score',
'sparsity',
'modelpath',
'modelfile',
'username',
'labgroup',
'public',
]
for c in cells:
for m in models:
dataRow = data.loc[(data.cellid == c) & (data.modelname == m)]
# if col is in the non_comp list, just copy the value if it exists
# then move on to the next iteration.
for col in columns:
if col in non_comp_columns:
try:
newData[col].loc[c,
m] = dataRow[col].values.tolist()[0]
continue
except:
newData[col].loc[c, m] = np.nan
continue
value = np.nan
newData[col].loc[c, m] = value
# If loop hits a continue, value will be left as NaN.
# Otherwise, will be assigned a value from data
# after passing all checks.
try:
value = dataRow[col].values.tolist()[0]
except Exception as e:
# Error should mean no value was recorded,
# so leave as NaN.
# No need to run outlier checks if value is missing.
print("No %s recorded for %s,%s" % (col, m, c))
continue
if not include_outliers:
# If outliers is false, run a bunch of checks based on
# measure and if a check fails, step out of the loop.
# Comments for each check are copied from
# from Narf_Analysis : compute_data_matrix
# "Drop r_test values below threshold"
a1 = (col == 'r_test')
b1 = (value < dataRow['r_floor'].values.tolist()[0])
a2 = (col == 'r_ceiling')
b2 = (dataRow['r_test'].values.tolist()[0] <
dataRow['r_floor'].values.tolist()[0])
a3 = (col == 'r_floor')
b3 = b1
if (a1 and b1) or (a2 and b2) or (a3 and b3):
continue
# "Drop MI values greater than 1"
a1 = (col == 'mi_test')
b1 = (value > 1)
a2 = (col == 'mi_fit')
b2 = (0 <= value <= 1)
if (a1 and b1) or (a2 and not b2):
continue
# "Drop MSE values greater than 1.1"
a1 = (col == 'mse_test')
b1 = (value > 1.1)
a2 = (col == 'mse_fit')
b2 = b1
if (a1 and b1) or (a2 and b2):
continue
# "Drop NLOGL outside normalized region"
a1 = (col == 'nlogl_test')
b1 = (-1 <= value <= 0)
a2 = (col == 'nlogl_fit')
b2 = b1
if (a1 and b1) or (a2 and b2):
continue
# TODO: is this still used? not listed in NarfResults
# "Drop gamma values that are too low"
a1 = (col == 'gamma_test')
b1 = (value < 0.15)
a2 = (col == 'gamma_fit')
b2 = b1
if (a1 and b1) or (a2 and b2):
continue
# TODO: is an outlier check needed for cohere_test
# and/or cohere_fit?
# If value existed and passed outlier checks,
# re-assign it to the proper DataFrame position
# to overwrite the NaN value.
newData[col].loc[c, m] = value
if only_fair:
# If fair is checked, drop all rows that contain a NaN value for
# every column.
for c in cells:
for m in models:
if newData.loc[c, m].isnull().values.all():
newData.drop(c, level='cellid', inplace=True)
break
# Swap the 0th and 1st levels so that modelname is the primary index,
# since most plots group by model.
newData = newData.swaplevel(i=0, j=1, axis=0)
return newData
def edit_analysis():
"""Take input from Analysis Editor modal and save it to the database.
Button : Edit Analysis
"""
user = get_current_user()
session = Session()
NarfAnalysis = Tables()['NarfAnalysis']
modTime = datetime.datetime.now().replace(microsecond=0)
eName = request.args.get('name')
eId = request.args.get('id')
eStatus = request.args.get('status')
eTags = request.args.get('tags')
eQuestion = request.args.get('question')
eAnswer = request.args.get('answer')
eLoad = request.args.get('load')
eMod = request.args.get('mod')
eFit = request.args.get('fit')
eTree = json.dumps([eLoad, eMod, eFit])
if eId == '__none':
checkExists = False
else:
checkExists = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == eId).first())
if checkExists:
a = checkExists
if (a.public or (user.labgroup in a.labgroup)
or (a.username == user.username)):
a.name = eName
a.status = eStatus
a.question = eQuestion
a.answer = eAnswer
a.tags = eTags
try:
a.lastmod = modTime
except:
a.lastmod = str(modTime)
a.modeltree = eTree
else:
log.info("You do not have permission to modify this analysis.")
return jsonify(success=("failed"))
# If it doesn't exist, add new sql alchemy object with the
# appropriate attributes, which should get assigned to a new id
else:
# TODO: Currently copies user's labgroup by default.
# Is that the behavior we want?
try:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=modTime,
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
except:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=str(modTime),
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
session.add(a)
addedName = a.name
session.commit()
session.close()
# After handling submissions, return user to main page so that it
# refreshes with new analysis included in list
return jsonify(success="Analysis %s saved successfully." % addedName)
def filter_cells(session, batch, cells, min_snr=0, min_iso=0, min_snri=0):
""" Returns a list of cells that don't meet the minimum snr/iso/snri
criteria specified. The calling function can then remove them from the
cell list if desired (ex: for cell in bad_cells, cell_list.remove(cell))
Arguments:
----------
batch : int
The batch number to query.
cells : list
A list of the cellids to be checked.
min_snr : float
The minimum signal to noise ratio desired.
min_iso : float
The minimum isolation value desired.
min_snri : float
The minimum signal to noise ratio index desired.
session : object
An open database session object for querying NarfBatches.
"""
bad_cells = []
session = Session()
NarfBatches = Tables()['NarfBatches']
for cellid in cells:
dbCriteria = (session.query(NarfBatches).filter(
NarfBatches.batch == batch).filter(
NarfBatches.cellid.ilike(cellid)).first())
if dbCriteria:
db_snr = min(dbCriteria.est_snr, dbCriteria.val_snr)
db_iso = dbCriteria.min_isolation
db_snri = dbCriteria.min_snr_index
a = (min_snr > db_snr)
b = (min_iso > db_iso)
c = (min_snri > db_snri)
if a or b or c:
bad_cells.append(cellid)
# Uncomment section below to include verbose output of
# why individual cells were 'bad'
#filterReason = ""
#if a:
# filterReason += (
# "min snr: %s -- was less than criteria: %s\n"
# %(db_snr, min_snr)
# )
#if b:
# filterReason += (
# "min iso: %s -- was less than criteria: %s\n"
# %(db_iso, min_iso)
# )
#if c:
# filterReason += (
# "min snr index: %s -- was less than criteria: %s\n"
# %(db_snri, min_snri)
# )
#print(
# "Removing cellid: %s,\n"
# "because: %s"
# %(cellid, filterReason)
# )
else:
print(
"No entry in NarfBatches for cellid: {0} in batch: {1}".format(
cellid, batch))
bad_cells.append(cellid)
print("Number of bad cells to snr/iso criteria: {0}".format(
len(bad_cells)))
print("Out of total cell count: {0}".format(len(cells)))
return bad_cells
def edit_analysis():
"""Take input from Analysis Editor modal and save it to the database.
Button : Edit Analysis
"""
user = get_current_user()
session = Session()
modTime = datetime.datetime.now().replace(microsecond=0)
eName = request.args.get('name')
eId = request.args.get('id')
eStatus = request.args.get('status')
eTags = request.args.get('tags')
eQuestion = request.args.get('question')
eAnswer = request.args.get('answer')
eTree = request.args.get('tree')
#TODO: add checks to require input inside form fields
# or allow blank so that people can erase stuff?
# Turned this off for now -- can re-enable when rule needs are more stable
# Make sure the keyword combination is valid using nems.keyword_rules
#try:
# mf = ModelFinder(eTree)
# for modelname in mf.modellist:
# keyword_test_routine(modelname)
#except Exception as e:
# return jsonify(success='Analysis not saved: \n' + str(e))
if eId == '__none':
checkExists = False
else:
checkExists = (session.query(NarfAnalysis).filter(
NarfAnalysis.id == eId).first())
if checkExists:
a = checkExists
if (a.public or (user.labgroup in a.labgroup)
or (a.username == user.username)):
a.name = eName
a.status = eStatus
a.question = eQuestion
a.answer = eAnswer
a.tags = eTags
try:
a.lastmod = modTime
except:
a.lastmod = str(modTime)
a.modeltree = eTree
else:
log.info("You do not have permission to modify this analysis.")
return jsonify(success=("failed"))
# If it doesn't exist, add new sql alchemy object with the
# appropriate attributes, which should get assigned to a new id
else:
# TODO: Currently copies user's labgroup by default.
# Is that the behavior we want?
try:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=modTime,
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
except:
a = NarfAnalysis(name=eName,
status=eStatus,
question=eQuestion,
answer=eAnswer,
tags=eTags,
batch='',
lastmod=str(modTime),
modeltree=eTree,
username=user.username,
labgroup=user.labgroup,
public='0')
session.add(a)
# For verifying correct logging - comment these out
# when not needed for testing.
#log.info("Added the following analysis to database:")
#log.info("------------------")
#log.info("name:"); log.info(a.name)
#log.info("question:"); log.info(a.question)
#log.info("answer:"); log.info(a.answer)
#log.info("status:"); log.info(a.status)
#log.info("tags:"); log.info(a.tags)
#log.info("model tree:"); log.info(a.modeltree)
#log.info("-----------------\n\n")
addedName = a.name
session.commit()
session.close()
# After handling submissions, return user to main page so that it
# refreshes with new analysis included in list
return jsonify(success="Analysis %s saved successfully." % addedName)
def update_results():
"""Update the results table after a batch, cell or model selection
is changed.
"""
user = get_current_user()
session = Session()
NarfResults = Tables()['NarfResults']
nullselection = """
MUST SELECT A BATCH AND ONE OR MORE CELLS AND
ONE OR MORE MODELS BEFORE RESULTS WILL UPDATE
"""
bSelected = request.args.get('bSelected')
cSelected = request.args.getlist('cSelected[]')
mSelected = request.args.getlist('mSelected[]')
colSelected = request.args.getlist('colSelected[]')
# If no batch, cell or model is selected, display an error message.
if (len(bSelected) == 0) or (not cSelected) or (not mSelected):
return jsonify(resultstable=nullselection)
# Only get numerals for selected batch.
bSelected = bSelected[:3]
# Use default value of 500 if no row limit is specified.
rowlimit = request.args.get('rowLimit', 500)
ordSelected = request.args.get('ordSelected')
# Parse string into appropriate sqlalchemy method
if ordSelected == 'asc':
ordSelected = asc
elif ordSelected == 'desc':
ordSelected = desc
sortSelected = request.args.get('sortSelected', 'cellid')
# Always add cellid and modelname to column lists,
# since they are required for selection behavior.
cols = [
getattr(NarfResults, 'cellid'),
getattr(NarfResults, 'modelname'),
]
cols += [
getattr(NarfResults, c) for c in colSelected
if hasattr(NarfResults, c)
]
# Package query results into a DataFrame
results = psql.read_sql_query(
Query(cols, session).filter(NarfResults.batch == bSelected).filter(
NarfResults.cellid.in_(cSelected)).filter(
NarfResults.modelname.in_(mSelected)).filter(
or_(
int(user.sec_lvl) == 9,
NarfResults.public == '1',
NarfResults.labgroup.ilike('%{0}%'.format(
user.labgroup)),
NarfResults.username == user.username,
)).order_by(ordSelected(getattr(
NarfResults, sortSelected))).limit(rowlimit).statement,
session.bind)
with pd.option_context('display.max_colwidth', -1):
resultstable = results.to_html(
index=False,
classes="table-hover table-condensed",
)
session.close()
return jsonify(resultstable=resultstable)
def make_jerb_json():
user = get_current_user()
session = Session()
# .all() returns a list of tuples, so it's necessary to pull the
# name elements out into a list by themselves.
analyses = (session.query(NarfAnalysis).filter(
or_(
int(user.sec_lvl) == 9,
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).order_by(asc(NarfAnalysis.id)).all())
analysislist = [a.name for a in analyses]
batchids = [
i[0] for i in session.query(NarfBatches.batch).distinct()
#.filter(or_(
# int(user.sec_lvl) == 9,
# NarfBatches.public == '1',
# NarfBatches.labgroup.ilike('%{0}%'.format(user.labgroup)),
# NarfBatches.username == user.username,
# ))
.all()
]
batchnames = []
for i in batchids:
name = (session.query(sBatch.name).filter(sBatch.id == i).first())
if not name:
batchnames.append('')
else:
batchnames.append(name.name)
batchlist = [(batch + ': ' + batchnames[i])
for i, batch in enumerate(batchids)]
batchlist.sort()
session.close()
s3 = boto3.resource('s3')
bucket = s3.Bucket('nemsdata')
jerb_json = {
'name': 'Analysis',
'children': [],
}
for i, analysis in enumerate(analysislist):
jerb_json['children'].append({'name': analysis, 'children': []})
jerb_json['children'][i]['children'].extend([{
'name':
'batch',
'children': [{
'name': batch,
'leaf': 1
} for batch in batchlist]
}, {
'name':
'models',
'children': [{
'name': model,
'leaf': 1
} for model in ['fake', 'model', 'list']]
}, {
'name':
'data',
'children': [{
'name': obj.key.strip('nems_in_cache/batch291/'),
'leaf': 1
} for i, obj in enumerate(
bucket.objects.filter(Prefix='nems_in_cache/batch291/'))
if i < 20]
}])
return jerb_json
def main_view():
"""Initialize the nems_analysis landing page.
Queries the database to get lists of available analyses, batches,
status filters, tag filters, and results columns.
Specifies defaults for results columns, row limit and sort column.
Returns:
--------
main.html : template
The landing page template rendered with variables for analysislist,
batchlist, collist, defaultcols, measurelist, defaultrowlimit,
sortlist, defaultsort, statuslist, and taglist.
"""
# TODO: figure out how to integrate sec_lvl/superuser mode
# maybe need to add sec_lvl column to analysis/batches/results?
# then can compare in query ex: if user.sec_lvl > analysis.sec_lvl
user = get_current_user()
session = Session()
db_tables = Tables()
NarfResults = db_tables['NarfResults']
NarfAnalysis = db_tables['NarfAnalysis']
NarfBatches = db_tables['NarfBatches']
sBatch = db_tables['sBatch']
# .all() returns a list of tuples, so it's necessary to pull the
# name elements out into a list by themselves.
analyses = (session.query(NarfAnalysis).filter(
or_(
int(user.sec_lvl) == 9,
NarfAnalysis.public == '1',
NarfAnalysis.labgroup.ilike('%{0}%'.format(user.labgroup)),
NarfAnalysis.username == user.username,
)).order_by(asc(NarfAnalysis.id)).all())
analysislist = [a.name for a in analyses]
analysis_ids = [a.id for a in analyses]
batchids = [
i[0] for i in session.query(NarfBatches.batch).distinct()
#.filter(or_(
# int(user.sec_lvl) == 9,
# NarfBatches.public == '1',
# NarfBatches.labgroup.ilike('%{0}%'.format(user.labgroup)),
# NarfBatches.username == user.username,
# ))
.all()
]
batchnames = []
for i in batchids:
name = (session.query(sBatch.name).filter(sBatch.id == i).first())
if not name:
batchnames.append('')
else:
batchnames.append(name.name)
batchlist = [(batch + ': ' + batchnames[i])
for i, batch in enumerate(batchids)]
batchlist.sort()
# Default settings for results display.
# TODO: let user choose their defaults and save for later sessions
# cols are in addition to cellid, modelname and batch,
# which are set up to be required
defaultcols = n_ui.cols
defaultrowlimit = n_ui.rowlimit
defaultsort = n_ui.sort
measurelist = n_ui.measurelist
statuslist = [
i[0] for i in session.query(NarfAnalysis.status).filter(
NarfAnalysis.name.in_(analysislist)).distinct().all()
]
# Separate tags into list of lists of strings.
tags = [
i[0].split(",") for i in session.query(NarfAnalysis.tags).filter(
NarfAnalysis.name.in_(analysislist)).distinct().all()
]
# Flatten list of lists into a single list of all tag strings
# and remove leading and trailing whitespace.
taglistbldupspc = [i for sublist in tags for i in sublist]
taglistbldup = [t.strip() for t in taglistbldupspc]
# Reform the list with only unique tags
taglistbl = list(set(taglistbldup))
# Finally, remove any blank tags and sort the list.
taglist = [t for t in taglistbl if t != '']
taglist.sort()
# Returns all columns in the format 'NarfResults.columnName,'
# then removes the leading 'NarfResults.' from each string
collist = ['%s' % (s) for s in NarfResults.__table__.columns]
collist = [s.replace('NarfResults.', '') for s in collist]
sortlist = copy.deepcopy(collist)
# Remove cellid and modelname from options toggles- make them required.
required_cols = n_ui.required_cols
for col in required_cols:
collist.remove(col)
# imported at top from PlotGenerator
plotTypeList = PLOT_TYPES
# imported at top from nems_web.run_scrits.script_utils
scriptList = scan_for_scripts()
session.close()
return render_template('main.html',
analysislist=analysislist,
analysis_ids=analysis_ids,
batchlist=batchlist,
collist=collist,
defaultcols=defaultcols,
measurelist=measurelist,
defaultrowlimit=defaultrowlimit,
sortlist=sortlist,
defaultsort=defaultsort,
statuslist=statuslist,
taglist=taglist,
plotTypeList=plotTypeList,
username=user.username,
seclvl=int(user.sec_lvl),
iso=n_ui.iso,
snr=n_ui.snr,
snri=n_ui.snri,
scripts=scriptList,
bokeh_version=bokeh_version)
def generate_plot_html():
session = Session()
plotType = request.args.get('plotType')
bSelected = request.args.get('bSelected')[:3]
mSelected = request.args.getlist('mSelected[]')
cSelected = request.args.getlist('cSelected[]')
measure = request.args.get('measure')
onlyFair = request.args.get('onlyFair')
if int(onlyFair):
onlyFair = True
else:
onlyFair = False
includeOutliers = request.args.get('includeOutliers')
if int(includeOutliers):
includeOutliers = True
else:
includeOutliers = False
# TODO: Re-do this to include any new criteria dynamically instead of
# hard-coding snr/iso/snri.
filterCriteria = {
'snr' : float(request.args.get('snr')),
'iso' : float(request.args.get('iso')),
'snri' : float(request.args.get('snri')),
}
# TODO: Looks like this is what NARF does, but not 100% sure.
# Always exclude negative values
for key in filterCriteria:
if filterCriteria[key] < 0:
filterCriteria[key] = 0
removalCount = 0
for cellid in cSelected:
dbCriteria = (
session.query(NarfBatches)
.filter(NarfBatches.batch == bSelected)
.filter(NarfBatches.cellid == cellid)
.all()
)
if dbCriteria:
if len(dbCriteria) > 1:
log.info(
"Multiple results found for cellid: %s in batch: %s"
%(cellid, bSelected)
)#!/usr/bin/env python3
min_snr = min(dbCriteria[0].est_snr, dbCriteria[0].val_snr)
min_isolation = dbCriteria[0].min_isolation
min_snr_index = dbCriteria[0].min_snr_index
a = (filterCriteria['snr'] > min_snr)
b = (filterCriteria['iso'] > min_isolation)
c = (filterCriteria['snri'] > min_snr_index)
if a or b or c:
filterReason = ""
if a:
filterReason += (
"min snr: %s -- was less than criteria: %s\n"
%(min_snr, filterCriteria['snr'])
)
if b:
filterReason += (
"min iso: %s -- was less than criteria: %s\n"
%(min_isolation, filterCriteria['iso'])
)
if c:
filterReason += (
"min snr index: %s -- was less than criteria: %s\n"
%(min_snr_index, filterCriteria['snri'])
)
#log.info(
# "Removing cellid: %s,\n"
# "because: %s"
# %(cellid, filterReason)
# )
cSelected.remove(cellid)
removalCount += 1
else:
log.info(
"No entry in NarfBatches for cellid: %s in batch: %s"
%(cellid, bSelected)
)
cSelected.remove(cellid)
removalCount += 1
log.info("Number of cells filtered due to snr/iso criteria: %d"%removalCount)
results = psql.read_sql_query(
session.query(NarfResults)
.filter(NarfResults.batch == bSelected)
.filter(NarfResults.cellid.in_(cSelected))
.filter(NarfResults.modelname.in_(mSelected))
.statement,
session.bind
)
log.debug("Results retrieved, with size: {}".format(results.size))
# get back list of models that matched other query criteria
results_models = [
m for m in
list(set(results['modelname'].values.tolist()))
]
# filter mSelected to match results models so that list is in the
# same order as on web UI
ordered_models = [
m for m in mSelected
if m in results_models
]
log.debug("Modelnames re-ordered and filtered to: {}".format(ordered_models))
Plot_Class = getattr(pg, plotType)
plot = Plot_Class(
data=results, measure=measure, models=ordered_models,
fair=onlyFair, outliers=includeOutliers,
)
log.debug("Plot successfully initialized")
if plot.emptycheck:
log.info('Plot checked empty after forming data array')
return jsonify(script='Empty',div='Plot')
else:
plot.generate_plot()
session.close()
if hasattr(plot, 'script') and hasattr(plot, 'div'):
return jsonify(script=plot.script, div=plot.div)
elif hasattr(plot, 'html'):
return jsonify(html=plot.html)
elif hasattr(plot, 'img_str'):
image = str(b64encode(plot.img_str))[2:-1]
return jsonify(image=image)
else:
return jsonify(script="Couldn't find anything ", div="to return")
