def _newTraitCategorical(sess, request, trt):
capKeys = [key for key in request.form.keys() if key.startswith("caption_")]
for key in capKeys:
caption = request.form.get(key)
value = request.form.get(key.replace("caption_", "value_"))
# If image provided we need to store it
imageURL = None
try:
imageURLFile = request.files[key.replace("caption_", "imgfile_")]
if imageURLFile:
sentFilename = secure_filename(imageURLFile.filename)
if allowed_file(sentFilename):
# Note, file are stored under a configured category image folder,
# and then with path <userName>/<trait id>/<value>.<original file extension>
subpath = os.path.join(app.config['CATEGORY_IMAGE_FOLDER'], sess.getProjectName(), str(trt.id))
if not os.path.exists(subpath):
os.makedirs(subpath)
fileExt = sentFilename.rsplit('.', 1)[1]
newFilename = '{0}.{1}'.format(value, fileExt)
imageURLFile.save(subpath + "/" + newFilename)
imageURL = app.config['CATEGORY_IMAGE_URL_BASE'] + "/" + sess.getProjectName() + "/" + str(trt.id) + "/" + newFilename
else:
util.flog("_newTraitCategorical bad file name")
except Exception, e:
util.flog("Exception in _newTraitCategorical: {0}".format(str(e)))
# Determine if this is a new category or an existing one.
# Trait categories are identified by the value (within a trait).
tcat = trt.getCategory(value)
newCat = tcat is None
if newCat:
# Add new trait category:
tcat = models.TraitCategory()
tcat.value = value
tcat.caption = caption
tcat.trait_id = trt.id
tcat.imageURL = imageURL
sess.db().add(tcat)
else:
util.flog("Existing category: cap:{0} value:{1}".format(caption, value))
# This is an existing category, update caption, or image URL if necessary:
if tcat.caption != caption:
tcat.caption = caption
# (Re)Set the image if necessary. There is a difficulty here.
# NB If an image was already set, and user doesn't change it, it will come back empty.
# And we don't want to set it to nothing in this case. The problem remains, however, as
# to how can the user explicitly set it to no image?
if imageURL is not None:
tcat.imageURL = imageURL
def urlAppCoolUploadCrashReport():
#-------------------------------------------------------------------------------------------------
# check file size?
# Handle a crash report upload from the app.
cfile = request.files.get('uploadedfile')
util.flog('urlAppCoolUploadCrashReport: filename {0}'.format(
cfile.filename))
if cfile and allowed_file(cfile.filename):
sentFilename = secure_filename(cfile.filename)
saveName = sentFilename
try:
# Need to check if file exists, if so postfix copy num to name so as not to overwrite:
fullPath = current_app.config[
'CRASH_REPORT_UPLOAD_FOLDER'] + saveName
cfile.save(fullPath)
return successResponse()
except Exception, e:
util.flog('failed save {0}'.format(
current_app.config['CRASH_REPORT_UPLOAD_FOLDER'] + saveName))
util.flog(e.__doc__)
util.flog(e.message)
return serverErrorResponse('Failed crash report upload : can'
't save')
def urlAppUploadTrialData(username, trial, dbc, token):
#-------------------------------------------------------------------------------------------------
# This version should return JSON!
# NB historical peculiarities here, this attow only used for upload nodes, or notes.
# And the format of the response differs between these two. Probably would be better
# with separate urls.
#
# Note used to have old_version, where the token was in the json not the url, see svn history
# for this code if necessary. Only relevant if old clients are still out there, but I think
# not. URLs sent and stored on client when this function was
# used should still be able to access this func (since the URL will match). But now
# we send out the URL for the new version.
jtrial = request.json
if not jtrial:
return Response('Bad or missing JSON')
util.flog("upload_trial:\n" + json.dumps(jtrial))
# Probably need a 'command' or 'type' field.
# different types will need different responses.
# Note old client would not have it, so perhaps below must stay
# Old clients may just send 'notes', we process that here in the manner they expect:
# MFK - so what do new clients do different? Nothing attow.
if 'notes' in jtrial: # We really should put these JSON names in a set of string constants somehow..
err = trial.addNodeNotes(token, jtrial[jTrialUpload['notes']])
if err is not None:
util.flog('addNodeNotes fail:{0}'.format(err))
return Response(err)
# All done, return success indicator:
return successResponse()
#
# Created Nodes:
# Process nodes created on the client. We need to create them on the server,
# and send back the ids of the new server versions. This needs to be idempotent,
# i.e. if a client sends a node more than once, it should only be created once
# on the server. This is managed by recording the token and the client node id
# for each created node (in the database).
#
if JTRL_NODES_ARRAY in jtrial:
# MFK - make this an array of objects, preferably same format as sent server to client.
clientLocalIds = jtrial[JTRL_NODES_ARRAY]
serverIds = []
serverRows = []
serverCols = []
# We have to return array of server ids to replace the passed in local ids.
# We need to record the local ids so as to be idempotent.
tokenObj = dal.Token.getToken(dbc, token, trial.id)
if tokenObj is None:
util.flog('urlAppUploadTrialData: invalid token')
return Response('invalid token')
for newid in clientLocalIds:
#print 'new id: {0}'.format(newid)
# Create node, or get it if it already exists:
node = dal.TokenNode.getOrCreateClientNode(dbc, tokenObj.id, newid,
trial.id)
serverIds.append(node.getId())
serverRows.append(node.getRow())
serverCols.append(node.getCol())
returnObj = {
'nodeIds': serverIds,
'nodeRows': serverRows,
'nodeCols': serverCols
}
return Response(json.dumps(returnObj),
mimetype='application/json') # prob need ob
# All done, return success indicator:
return successResponse()
def urlAppUploadPhoto(username, trial, dbc, traitid, token):
#-------------------------------------------------------------------------------------------------
# Handle a photo upload from the app.
# These are uniquely identified by dbusername/trial/trait/token/seqNum/sampleNum.
# These are all provided in the url except for seqNum and sampleNum which come
# (out-of-band) as form parameters. In addition the score metadata is also provided
# in the form parameters. This enables us to create the datum record in this function.
# Note originally we only saved the photo here, and the client would separately upload
# the traitInstance containing all the meta data as for a non-photo traitInstance.
# This was problematic because until both the metadata, and the photo itself are both
# uploaded we have not successfully uploaded. Hence it is better done in a single
# transaction (this func). Currently, the client will first upload a trait instance
# WITHOUT any datums, to ensure the traitInstance is created on the server, and to
# set its creation date. If we didn't do this we would have to upload the creation
# date with every photo (or at least the first one) so as to allow us to create the
# traitInstance record in this function (if it was not already present).
#
# UPDATE - Attow the creation date is sent with each photo, and the client does NOT
# first create an empty ti.
#
# Note the node Id is uploaded in the file name (this should be same as server node id).
# But in addition it is (now) also uploaded as a form parameter, this reduces our
# dependence on it being in the file name (which may change).
# The photos are saved in the PHOTO_UPLOAD_FOLDER folder, with the name encoding
# all the relevant info:
# '{0}_{1}_{2}_{3}_{4}_{5}_{6}.jpg'.format(dbusername, trialId, traitId, nodeId, token, seqNum, sampNum)
#
seqNum = request.args.get(TI_SEQNUM, '')
sampNum = request.args.get(TI_SAMPNUM, '')
timestamp = request.args.get(DM_TIMESTAMP, '')
userid = request.args.get(DM_USERID, '')
gpslat = request.args.get(DM_GPS_LAT, '')
gpslong = request.args.get(DM_GPS_LONG, '')
nodeId = request.args.get(DM_NODE_ID_CLIENT_VERSION, '')
dayCreated = request.args.get(TI_DAYCREATED, '')
file = request.files.get('uploadedfile')
util.flog(
'urlAppUploadPhoto:node {0}, seq {1} samp {2} filename {3}'.format(
nodeId, seqNum, sampNum, file.filename))
if file and allowed_file(file.filename):
sentFilename = secure_filename(file.filename)
saveName = dal.photoFileName(username, trial.id, traitid, int(nodeId),
token, seqNum, sampNum)
try:
# Need to check if file exists, if so postfix copy num to name so as not to overwrite:
fullPath = current_app.config['PHOTO_UPLOAD_FOLDER'] + saveName
print 'fullPath {}'.format(
current_app.config['PHOTO_UPLOAD_FOLDER'] + saveName)
base = os.path.splitext(fullPath)[0]
ext = os.path.splitext(fullPath)[1]
tryAgain = os.path.isfile(fullPath)
i = 1
while tryAgain:
fullPath = '{0}_c{1}{2}'.format(base, i, ext)
i += 1
tryAgain = os.path.isfile(fullPath)
file.save(fullPath)
except Exception, e:
util.flog('failed save {0}'.format(
current_app.config['PHOTO_UPLOAD_FOLDER'] + saveName))
util.flog(e.__doc__)
util.flog(e.message)
return serverErrorResponse('Failed photo upload : can' 't save')
# Now save datum record:
# get TI - this should already exist, which is why we can pass in 0 for dayCreated
# MFK note this outer if below is to support old versions of the app, to allow them to
# upload their photos in the old way. It should be removed eventually..
if nodeId is not None and len(nodeId) > 0:
dbTi = dal.getOrCreateTraitInstance(dbc, traitid, trial.id, seqNum,
sampNum, dayCreated, token)
if dbTi is None:
return serverErrorResponse(
'Failed photo upload : no trait instance')
res = dal.AddTraitInstanceDatum(dbc, dbTi.id, dbTi.trait.datatype,
nodeId, timestamp, userid,
gpslat, gpslong,
os.path.basename(fullPath))
if res is None:
return successResponse()
else:
return serverErrorResponse(
'Failed photo upload : datum create fail')
else:
util.flog('urlAppUploadPhoto: no nodeId, presumed old app version')
return successResponse()
def serverErrorResponse(msg):
util.flog(msg)
response = Response(msg)
response.status = '500 {0}'.format(msg)
return response
# Get json fields:
try:
dayCreated = jti["dayCreated"]
seqNum = jti["seqNum"]
sampleNum = jti["sampleNum"]
except Exception, e:
return Response('Missing required traitInstance field: ' + e.args[0] +
trial.name)
try:
aData = jti["data"]
except Exception, e:
aData = None
# Log upload, but don't output json.dumps(jti), as it can be big:
util.flog(
"urlAppUploadTraitData from {0}: dc:{1}, seq:{2}, samp:{3}".format(
token, dayCreated, seqNum, sampleNum,
"None" if aData is None else len(aData)))
# MFK: A problem here in that ideally we don't want to create empty scoresets.
# The photo upload code, relies on being able to create an empty scoreset on the
# server prior to uploading the pictures. Actually, that's now not the case, but
# there may be versions of the app out there for a while that do require an empty
# set to be created. So for the moment, we'll limit this to photo traits only,
# and then remove that when we're confident all apps are updated:
if (aData is None or len(aData) <= 0) and dal.getTrait(
dbc, traitid).datatype != T_PHOTO:
return successResponse()
# Get/Create trait instance:
dbTi = dal.getOrCreateTraitInstance(dbc, traitid, trial.id, seqNum,
sampleNum, dayCreated, token)
if dbTi is None:
def traitDetailsPageHandler(sess, request, trial, trialId, traitId):
#===========================================================================
# Handles both GET and POST for page to display/modify details for a trait.
#
# MFK Note overlap with code from trait creation.
# Needs rework, most models calls here should be from trial object, not statics
#
projId = sess.getProjectId()
trt = models.getTrait(sess.db(), traitId)
if trt is None:
util.flog("No trait in traitDetailsPageHandler, trialId: {}, traitId: {}".format(trialId, traitId))
return errorScreenInSession('Cannot find trait')
trlTrt = models.getTrialTrait(sess.db(), trialId, traitId)
if trlTrt is None:
util.flog("No trialTrait in traitDetailsPageHandler, trialId: {}, traitId: {}".format(trialId, traitId))
return errorScreenInSession('Cannot find trial trait')
title = 'Trial: ' + trial.name + ', Trait: ' + trt.caption
comparatorCodes = [
["gt", "Greater Than", 1],
["ge", "Greater Than or Equal to", 2],
["lt", "Less Than", 3],
["le", "Less Than or Equal to", 4]
]
if request.method == 'GET':
###########################################################################
# Form fields applicable to all traits:
###
formh = fpUtil.htmlLabelValue('Trait', trt.caption) + '<br>'
formh += fpUtil.htmlLabelValue('Type', TRAIT_TYPE_NAMES[trt.datatype])
formh += fpUtil.htmlHorizontalRule()
formh += fpUtil.htmlLabelValue('Description',
'<input type="text" size=96 name="description" value="{0}">'.format(trt.description))
# Trait barcode selection:
# Note it doesn't matter if a sysTrait, since the barcode is stored in trialTrait
attSelector = '<select name="bcAttribute" id="bcAttribute">'
attSelector += '<option value="none"><Choose Attribute></option>'
atts = trial.getAttributes()
for att in atts:
attSelector += '<option value="{0}" {2}>{1}</option>'.format(
att.id, att.name, "selected='selected'" if att.id == trlTrt.barcodeAtt_id else "")
attSelector += '</select>'
formh += '<br>' + fpUtil.htmlLabelValue('Barcode for Scoring', attSelector)
# Vars that may be set by trait specifics, to be included in output:
preform = ''
onsubmit = ''
if trt.datatype == T_CATEGORICAL:
# Note the intended policy: Users may modify the caption or image of an existing
# category, but not change the numeric value. They may add new categories.
# The reasoning is that they should remove existing categories (as identified
# by the value) because there may be data collected already with the values
# which would then become undefined, or perhaps ambiguous. We could have a look
# up and allow removal if there is no data, but we cannot be sure data will
# not come in in the future referencing current values (unless we know trial
# has never been downloaded). In any case this limited functionality is better
# than none.
#####
## Setup javascript to manage the display/modification of the categories.
#####
# Retrieve the categories from the database and make of them a javascript literal:
catRecs = trt.categories
catObs = ''
for cat in catRecs:
if catObs: catObs += ','
catObs += '{{caption:"{0}", imageURL:"{1}", value:{2}}}'.format(cat.caption, cat.imageURL, cat.value)
jsRecDec = '[{0}]'.format(catObs)
formh += '<div id="traitTypeSpecificsDiv"></div>\n'
formh += '<script src="{0}"></script>\n'.format(url_for('static', filename='newTrait.js'))
formh += """<script type="text/javascript">
jQuery(function(){{fpTrait.setTraitFormElements('traitTypeSpecificsDiv', '3', {0});}});
</script>""".format(jsRecDec)
onsubmit = "return fpTrait.validateTraitTypeSpecific('traitTypeSpecificsDiv', '{0}')".format(trt.datatype)
elif trt.datatype == T_INTEGER or trt.datatype == T_DECIMAL:
#
# Generate form on the fly. Could use template but there's lots of variables.
# Make this a separate function to generate html form, so can be used from
# trait creation page. MFK - maybe not relevant now for trait creation page,
# as we don't put the min/max or validation options there (in case it's a sys
# trait, and these things should be able to vary between instances of the trait).
#
ttn = models.GetTrialTraitNumericDetails(sess.db(), traitId, trialId)
# Min and Max:
# need to get decimal version if decimal. Maybe make ttn type have getMin/getMax func and use for both types
minText = ""
if ttn and ttn.min is not None:
minText = "value='{:f}'".format(ttn.getMin())
maxText = ""
if ttn and ttn.max is not None:
maxText = "value='{:f}'".format(ttn.getMax())
minMaxBounds = "<p>Minimum: <input type='text' name='min' id=tdMin {0}>".format(minText)
minMaxBounds += "<p>Maximum: <input type='text' name='max' id=tdMax {0}><br>".format(maxText);
# Parse condition string, if present, to retrieve comparator and attribute.
# Format of the string is: ^. <2_char_comparator_code> att:<attribute_id>$
# The only supported comparison at present is comparing the score to a
# single attribute.
# NB, this format needs to be in sync with the version on the app. I.e. what
# we save here, must be understood on the app.
# MFK note attribute id seems to be stored as text in cond string, will seems
# not ideal. Probably should be a field in the table trialTraitNumeric.
# Note that the same issue applies in the app database There is one advantage
# I see to having a string is that we can change what is stored without requiring
# a database structure change. And db structure changes on the app require
# a database replace on the app.
atId = -1
op = ""
if ttn and ttn.cond is not None:
tokens = ttn.cond.split() # [["gt", "Greater than", 0?], ["ge"...]]?
if len(tokens) != 3:
return "bad condition: " + ttn.cond
op = tokens[1]
atClump = tokens[2]
atId = int(atClump[4:])
# Show available comparison operators:
valOp = '<select name="validationOp" id="tdCompOp">'
valOp += '<option value="0"><Choose Comparator></option>'
for c in comparatorCodes:
valOp += '<option value="{0}" {2}>{1}</option>'.format(
c[2], c[1], 'selected="selected"' if op == c[0] else "")
valOp += '</select>'
# Attribute list:
attListHtml = '<select name="attributeList" id="tdAttribute">'
attListHtml += '<option value="0"><Choose Attribute></option>'
for att in atts:
if att.datatype == T_DECIMAL or att.datatype == T_INTEGER:
attListHtml += '<option value="{0}" {2}>{1}</option>'.format(
att.id, att.name, "selected='selected'" if att.id == atId else "")
attListHtml += '</select>'
# javascript form validation
# NEED TO Check that min and max are valid int or decimal
# Check that if one of comp and att chosen both are
# Note this is the same validation for integer and decimal. So integer
# will allow decimal min/max. Could be made strict, but I'm not sure this is bad.
script = """
<script>
function isValidDecimal(inputtxt) {
var decPat = /^[+-]?[0-9]+(?:\.[0-9]+)?$/g;
return inputtxt.match(decPat);
}
function validateTraitDetails() {
// Check min and max fields:
/* It should be OK to have no min or max:
if (!isValidDecimal(document.getElementById("tdMin").value)) {
alert('Invalid value for minimum');
return false;
}
if (!isValidDecimal(document.getElementById("tdMax").value)) {
alert('Invalid value for maximum');
return false;
}
*/
// Check attribute/comparator fields, either both or neither present:
var att = document.getElementById("tdAttribute").value;
var comp = document.getElementById("tdCompOp").value;
var attPresent = (att !== null && att !== "0");
var compPresent = (comp !== null && comp !== "0");
if (attPresent && !compPresent) {
alert("Attribute selected with no comparator specified, please fix.");
return false;
}
if (!attPresent && compPresent) {
alert("Comparator selected with no attribute specified, please fix.");
return false;
}
return true;
}
</script>
"""
formh += minMaxBounds
formh += '<p>Integer traits can be validated by comparison with an attribute:'
formh += '<br>Trait value should be ' + valOp + attListHtml
preform = script
onsubmit ='return validateTraitDetails()'
elif trt.datatype == T_STRING:
tts = models.getTraitString(sess.db(), traitId, trialId)
patText = "value='{0}'".format(tts.pattern) if tts is not None else ""
formh += "<p>Pattern: <input type='text' name='pattern' id=tdMin {0}>".format(patText)
formh += ('\n<p><input type="button" value="Cancel"' +
' onclick="location.href=\'{0}\';">'.format(fpUtil.fpUrl('urlTrial', sess, trialId=trialId)))
formh += '\n<input type="submit" value="Submit">'
return dp.dataPage( \
content=preform + fpUtil.htmlForm(formh, post=True, onsubmit=onsubmit, multipart=True),
title='Trait Validation', trialId=trialId)
##################################################################################################
if request.method == 'POST':
### Form fields applicable to all traits:
# Trait barcode selection:
#MFK sys traits? barcode field is an nodeAttribute id but this is associated with a trial
# we either have to move it to trialTrait, or make all trial traits non system traits.
barcodeAttId = request.form.get('bcAttribute') # value should be valid attribute ID
if barcodeAttId == 'none':
trlTrt.barcodeAtt_id = None
else:
trlTrt.barcodeAtt_id = barcodeAttId
trt.description = request.form.get('description')
#
# Trait type specific stuff:
#
if trt.datatype == T_CATEGORICAL:
_newTraitCategorical(sess, request, trt)
elif trt.datatype == T_INTEGER or trt.datatype == T_DECIMAL: # clone of above remove above when integer works with numeric
op = request.form.get('validationOp') # value should be [1-4], see comparatorCodes
if not re.match('[0-4]', op):
return "Invalid operation {0}".format(op) # should be some function to show error page..
at = request.form.get('attributeList') # value should be valid attribute ID
# Check min/max:
vmin = request.form.get('min')
if len(vmin) == 0:
vmin = None
vmax = request.form.get('max')
if len(vmax) == 0:
vmax = None
# Get existing trialTraitNumeric, or create new one if none:
ttn = models.GetTrialTraitNumericDetails(sess.db(), traitId, trialId)
newTTN = ttn is None
if newTTN:
ttn = models.TrialTraitNumeric()
ttn.trial_id = trialId
ttn.trait_id = traitId
ttn.min = vmin
ttn.max = vmax
if int(op) > 0 and int(at) > 0:
ttn.cond = ". " + comparatorCodes[int(op)-1][0] + ' att:' + at
if newTTN:
sess.db().add(ttn)
elif trt.datatype == T_STRING:
newPat = request.form.get('pattern')
tts = models.getTraitString(sess.db(), traitId, trialId)
if len(newPat) == 0:
newPat = None
# delete tts if not needed:
if not newPat:
if tts:
sess.db().delete(tts)
else:
if tts:
tts.pattern = newPat
else:
tts = models.TraitString(trait_id=traitId, trial_id=trialId, pattern=newPat)
sess.db().add(tts)
sess.db().commit()
return redirect(fpUtil.fpUrl('urlTrial', sess, trialId=trialId))