def getLayoutList(mapId, appCtx):
# Get the layout IDs of a map.
# Read the layouts file to get the layouts.
mapPath = os.path.join(appCtx.dataRoot, 'view', mapId)
layoutList = []
try:
with open(os.path.join(mapPath, 'layouts.tab'), 'r') as f:
f = csv.reader(f, delimiter='\t')
for row in f:
layoutList.append(row[0])
except Exception as e:
print 'exception: ', str(e)
# There may be a file under the older name of "matrixnames.tab".
try:
with open(os.path.join(mapPath, 'matrixnames.tab'), 'r') as f:
f = csv.reader(f, delimiter='\t')
for row in f:
layoutList.append(row[0])
except Exception as e:
raise ErrorResp('With the layout list: ' + str(e), 500)
# No layouts means we're done.
if len(layoutList) < 1:
raise ErrorResp('Layouts for map not found.', 404)
return layoutList
def oldApi(dataId, request):
# If the client does not include a file property in the post
# there is nothing to upload. So bail.
if 'file' not in request.files:
raise ErrorResp('No file property provided for upload', 400)
file = request.files['file']
# If a browser user does not select a file, the browser submits an
# empty file property. So bail.
if file.filename == '':
raise ErrorResp('No file provided for upload', 400)
filename = validate.cleanFileName(file.filename)
appCtx = getAppCtx()
path = os.path.join(appCtx.dataRoot, dataId)
# Make the directories if they are not there.
try:
os.makedirs(path[:path.rfind('/')], 0770)
except Exception:
pass
# Save the file
try:
file.save(path)
except Exception:
raise ErrorResp('Unable to save file: ' + filename, 500)
return filename
def _validateInteger (name, data, required=False):
if name in data:
try:
val = int(data[name])
except ValueError:
raise ErrorResp(name + ' parameter must be an integer')
elif required: # name is not in data
raise ErrorResp(name + ' parameter missing or malformed')
def validatePost():
if request.headers['Content-Type'] != 'application/json':
raise ErrorResp('Content-Type must be application/json')
try:
dataIn = request.get_json()
except:
raise ErrorResp('Post content is invalid JSON')
# Validate an optional email address so we know who a job belongs to.
validate.email(dataIn)
return dataIn
def add(email, token, authGroup, major, minor, request):
# Add a file to the upload directory and its info to the database.
print '### uploadFile.add()'
# Validate parms and clean the file name.
(fileName, fileObj) = _validateUploadFile(
{
'email': email,
'token': token,
'authGroup': authGroup,
'major': major,
'minor': minor,
}, request)
# Authenticate and authorize.
appCtx = getAppCtx()
_auth(email, token, appCtx)
# Add the file info to the database.
db = UploadDb(appCtx.uploadDbPath)
id = _addDb(email, authGroup, major, minor, fileName, fileObj.size, db)
# Save the file.
(status, msg) = _saveFile(fileObj, major, minor, fileName,
ctx.uploadFilePath)
# Update the database status.
if status == 'success':
db.updateStatus(id, db.success)
else:
db.updateStatus(id, db.error)
raise ErrorResp('Unable to save file: ' + fileName + ': ' + msg, 500)
return (id, fileName) # TODO: do we need the ID here?
def getAttrById(attrId, mapId, appCtx):
# Get an attribute's values by ID.
attrIndex = getAttrFilename(attrId, mapId, appCtx)
# No attr means we're done.
if attrIndex == None:
raise ErrorResp('Attribute not found.', 404)
mapPath = os.path.join(appCtx.dataRoot, 'view', mapId)
# Get the dataType.
try:
path = os.path.join(mapPath, 'Layer_Data_Types.tab')
dataType = None
with open(path, 'r') as f:
f = csv.reader(f, delimiter='\t')
# Convert the tsv to a dictionary arrays.
types = {
}
for row in f:
type = row[0]
for attr in row:
if attr == attrId and type != 'FirstAttribute':
dataType = type
break
except Exception as e:
raise ErrorResp('With finding data type for attribute: ' + str(e), 404)
# Get the values for this attr.
try:
path = os.path.join(mapPath, attrIndex)
with open(path, 'r') as f:
f = csv.reader(f, delimiter='\t')
# Convert the tsv to a dictionary of two arrays.
nodes = []
values = []
for j, row in enumerate(f.__iter__()):
nodes.append(row[0])
values.append(stringToFloatOrInt(row[1], dataType))
except Exception as e:
raise ErrorResp('With retrieving the attribute data: ' + str(e), 404)
return { 'dataType': dataType, 'nodes': nodes, 'values': values }
def _dataIdOrUrl (dataId, url, data, required=False):
if dataId in data:
_validateString(dataId, data)
_validatePathName(data[dataId], dataId)
elif url in data:
_validateString(url, data)
elif required:
raise ErrorResp(
dataId + ' or ' + url + ' parameter missing or malformed')
def deleteMap(mapId, userEmail=None, userRole=[]):
# Verify this is from the allowed view server
if not request.environ['REMOTE_ADDR'] in appCtx.viewServerAddrs and \
not request.environ['HTTP_X_FORWARDED_FOR'] in appCtx.viewServerAddrs:
raise ErrorResp('', 404)
raise SuccessResp(
projectEdit.delete(mapId, userEmail, _urlParmToList(userRole), appCtx))
def _highlightAttrNodeInner(data, appCtx):
# Validate data
validate.map(data, required=True)
validate.layout(data)
validate.attributes(data)
validate.nodes(data)
if not 'attributes' in data and not 'nodes' in data:
raise ErrorResp('One or both of the parameters, ' + \
'"attributes" and "nodes" must be provided.', 400)
# Fill the required state for the bookmark.
state = { 'page': 'mapPage', 'project': data['map'] + '/' }
if 'layout' in data:
state['layoutName'] = data['layout']
if 'nodes' in data:
# Create a dynamic attribute of the node list for the state.
attrData = {}
for node in data['nodes']:
attrData[node] = 1
state['dynamicAttrs'] = {
'yourNodes': {
'data': attrData,
'dataType': 'binary',
}
}
state['shortlist'] = ['yourNodes']
# Add the filter so the given nodes are highlighted on any attributes.
state['shortEntry.filter'] = {
'yourNodes': {
'by': 'category',
'value': [1],
}
}
if 'attributes' in data:
# Add the provided attributes to the shortlist state.
if 'nodes' in data:
state['shortlist'] += data['attributes']
else:
state['shortlist'] = data['attributes']
# Set the active attr state to the first attribute in the data.
state['activeAttrs'] = [data['attributes'][0]]
else:
# Without attrs provided, set the active attr state to the new attr of
# nodes just created.
state['activeAttrs'] = [state['shortlist'][0]]
return state
def updateColor():
# Verify this is from the allowed view server.
# TODO why not use the view server name and certificate here,
# rather than the IP addr?
if not request.environ['REMOTE_ADDR'] in appCtx.viewServerAddrs and \
not request.environ['HTTP_X_FORWARDED_FOR'] in appCtx.viewServerAddrs:
raise ErrorResp('', 404)
raise SuccessResp(projectEdit.updateColor(validatePost(), appCtx))
def map(data, required):
_validateString('map', data, required)
# Is the name file-safe?
val = data['map']
slashCount = val.count('/')
if slashCount > 1:
raise ErrorResp('map IDs may not contain more than one slash')
else:
_validateFileName(val, 'map', allowSlash=True)
def getStatus(id, queuePath):
# Retrieve the status and result of the given job ID.
# @param id: the job ID
# @param queuePath: the job queue path
# @returns: a dict of the form: {'status': <status>, 'result': <dict>}
# where result is None if the job is not found;
# only Success and Error may have an optional result; if
# there is no result, no result property is returned
statusResult = JobQueue(queuePath).getStatus(id)
if statusResult == None:
raise ErrorResp('unknown job ID of: ' + str(id))
return statusResult
def _validateUploadFile(data, request):
# If the client does not include a file property in the post
# there is nothing to upload. So bail.
if 'file' not in request.files:
raise ErrorResp('No file property provided for upload', 400)
# If a browser user does not select a file, the browser submits an
# empty file property. So bail.
fileObj = request.files['file']
if fileObj.filename == '':
raise ErrorResp('No file provided for upload', 400)
validate.emailSingleRequired(data)
validate.token(data)
validate.authGroup(data)
validate.major(data)
validate.minor(data)
# Remove any strange characters from the fileName
cleanFileName = validate.cleanFileName(file.filename)
return (cleanFileName, fileObj)
def _validateStringChars(val, name):
'''
Look for any non-printable characters in a string value, non-printables are
ascii decimal codes 0-31 and 127-255.
@param val: the string value
@param name: the name of the parameter
@return: nothing or raise an ErrorResp
'''
regex = r'[\x00-\x1f\x7f-\xff]'
search = re.search(regex, val)
if not search == None:
raise ErrorResp(name +
' parameter should only contain printable characters')
def getAttrList(mapId, appCtx):
# Read the layer summary file to find the attr IDs.
mapPath = os.path.join(appCtx.dataRoot, 'view', mapId)
attrList = []
try:
with open(os.path.join(mapPath, 'layers.tab'), 'r') as f:
f = csv.reader(f, delimiter='\t')
for row in f:
attrList.append(row[0])
except Exception as e:
raise ErrorResp('Retrieving attribute list: ' + str(e), 500)
return attrList
def _validateParms(data):
'''
Validate the query.
@param data: data received in the http post request
@return: nothing
'''
# Basic checks on required parameters
validate.map(data, True)
validate.layout(data, True)
if 'nodes' not in data:
raise ErrorResp('nodes parameter missing or malformed')
if not isinstance(data['nodes'], dict):
raise ErrorResp('nodes parameter should be a dictionary')
if len(data['nodes'].keys()) < 1:
raise ErrorResp('there are no nodes in the nodes dictionary')
# Basic checks on optional parameters
validate.email(data)
validate.viewServer(data)
if 'neighborCount' in data and \
(not isinstance(data['neighborCount'], int) or \
data['neighborCount'] < 1):
raise ErrorResp('neighborCount parameter should be a positive integer')
def _validateString(name, data, required=False, arrayAllowed=False):
'''
Validate a string or an array of strings.
@param name: the name of the property in the data
@param data: the object in which the property resides
@param required: this property is required in the data, optional,
defaults to false
@param arrayAllowed: an array of strings are allowed for this property,
optional, defaults to false
@return: nothing or raise an ErrorResp
'''
if name in data:
val = data[name]
if isinstance(val, types.StringTypes):
if len(val) < 1:
raise ErrorResp(name +
' parameter must have a string length greater than one')
_validateStringChars(val, name)
else:
# This is not a string, but maybe an array.
if arrayAllowed:
if not isinstance(val, list):
raise ErrorResp(name +
' parameter should be a string or an array of strings')
# Check each string in the array
for value in val:
if not isinstance(value, types.StringTypes):
raise ErrorResp(name +
' parameter should be a string or an array of strings')
_validateStringChars(value, name)
else:
raise ErrorResp(name + ' parameter should be a string')
elif required: # name is not in data
raise ErrorResp(name + ' parameter missing or malformed')
def getAttrFilename(attrId, mapId, appCtx):
# Read the layer summary file to find the attr file name.
mapPath = os.path.join(appCtx.dataRoot, 'view', mapId)
attrIndex = None
try:
with open(os.path.join(mapPath, 'layers.tab'), 'r') as f:
f = csv.reader(f, delimiter='\t')
for row in f:
if row[0] == attrId:
attrIndex = row[1]
break
except Exception as e:
raise ErrorResp('With the attribute summary: ' + str(e), 500)
return attrIndex
def getReflectionMetadata(mapId):
"""Get data needed for client have reflection available."""
majorId, minorId = getProjMajor(mapId), getProjMinor(mapId)
reflectJson = getReflectJson()
dataTypes = getDataTypes(majorId, reflectJson)
toMapIds = getToMapIds(mapId, reflectJson)
metadata = {
"dataTypes": dataTypes,
"toMapIds": toMapIds
}
if toMapIds is None and dataTypes is None:
raise ErrorResp(
"Reflection meta data for this map is not found.",
204
)
return metadata
def getFirstAttr(mapId, appCtx):
# Get the dataType file which also contains the first attribute.
try:
mapPath = os.path.join(appCtx.dataRoot, 'view', mapId)
path = os.path.join(mapPath, 'Layer_Data_Types.tab')
first = None
with open(path, 'r') as f:
f = csv.reader(f, delimiter='\t')
for row in f:
type = row[0]
for attr in row:
if type == 'FirstAttribute':
first = row[1]
break
except Exception as e:
raise ErrorResp('With finding default first attribute: ' + str(e), 404)
return first
def _getProjectRoles(project, viewDir):
roles = None
jsonStr = ''
try:
# Look for a meta file containing roles with access to this project.
with open(os.path.join(viewDir, project, 'meta.json'), 'rU') as f:
jsonStr = f.read()
except:
# There is no meta data for this project, so no roles.
return []
try:
meta = json.loads(jsonStr)
roles = meta['role']
except:
# Bad json in the meta data.
raise ErrorResp('bad json in meta data for map: ' + project)
return []
return _rolesToList(roles)
def _saveFile(fileObj, major, minor, fileName, uploadPath):
# Make the full path to save the file.
path = makeFullPath(major, fileName, minor, uploadPath)
# Make the directories if they are not there, read/writable by group.
try:
os.makedirs(path[:path.rfind('/')], 0770)
except Exception:
pass
# Save the file.
try:
fileObj.save(path)
except Exception(error):
return ('error', error)
db.updateStatus(id, db.error)
raise ErrorResp('Unable to save file: ' + fileName, 500)
return ('success', None)
def _validateFileName (dirty, name, allowSlash=False):
'''
check to be sure this is a file-safe name without any problem characters
Valid characters:
a-z, A-Z, 0-9, dash (-), dot (.), underscore (_)
All other characters are replaced with underscores.
@param dirty: the string to check
@param name: the data property name
@param allowSlash: allow a slash (/) in the string for paths
@return: nothing or raise an ErrorResp
'''
msg = name + ' parameter may only contain the characters:' + \
' a-z, A-Z, 0-9, dash (-), dot (.), underscore (_)'
if allowSlash:
regex = fileSlashRegEx
msg += ', slash (/)'
else:
regex = fileRegEx
search = re.search(regex, dirty)
if not search == None:
raise ErrorResp(msg)
def dataRouteInner(dataId, ok404=False):
# Not using flask's send_file() as it mangles files larger than 32k.
# Not using flask's GET because we need to capture 404-file-not-found
# errors in the case that is ok with the client and return success.
try:
with open(os.path.join(appCtx.dataRoot, dataId)) as f:
data = f.read()
result = Response(
data,
mimetype='text/csv',
headers={'Content-disposition': 'attachment'})
except IOError:
if ok404:
result = Response(
'404',
mimetype='text/csv',
headers={'Content-disposition': 'attachment'})
raise SuccessRespNoJson(result)
else:
raise ErrorResp('File not found or other IOError', 404)
raise SuccessRespNoJson(result)
def neighborCount (data):
name = 'neighborCount'
_validateInteger (name, data)
if name in data:
if data[name] < 1 or data[name] > 30:
raise ErrorResp('neighborCount parameter must be within the range, 1-30')
def authorize(project, userEmail, userRole, viewDir):
# Check for authorization to edit this map.
auth = projectList.authorize(project, userEmail, userRole, viewDir)
if auth['authorized'] != 'edit':
raise ErrorResp('', 404)
def byteSize (data):
name = 'byteSize'
_validateInteger (name, data)
if name not in data:
raise ErrorResp('byteSize parameter is required')