def wordcloud():
"""
Handles the functionality on the visualisation page -- a prototype for displaying
single word cloud graphs.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'cloudoption' not in session:
session['cloudoption'] = constants.DEFAULT_CLOUD_OPTIONS
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
# there is no wordcloud option so we don't initialize that
return render_template('wordcloud.html', labels=labels)
if request.method == "POST":
# "POST" request occur when html form is submitted (i.e. 'Get Dendrogram', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = utility.generateJSONForD3(fileManager, mergedSet=True)
# Create a list of column values for the word count table
from operator import itemgetter
terms = sorted(JSONObj["children"], key=itemgetter('size'), reverse=True)
columnValues = []
for term in terms:
rows = [term["name"], term["size"]]
columnValues.append(rows)
session_functions.cacheCloudOption()
return render_template('wordcloud.html', labels=labels, JSONObj=JSONObj, columnValues=columnValues)
def csvgenerator():
"""
Handles the functionality on the csvgenerator page. It analyzes the texts to produce
and send various frequency matrices.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
if 'csvoptions' not in session:
session['csvoptions'] = constants.DEFAULT_CSV_OPTIONS
labels = fileManager.getActiveLabels()
return render_template('csvgenerator.html', labels=labels)
if 'get-csv' in request.form:
#The 'Generate and Download Matrix' button is clicked on csvgenerator.html.
session_functions.cacheCSVOptions()
savePath, fileExtension = fileManager.generateCSV()
return send_file(savePath, attachment_filename="frequency_matrix"+fileExtension, as_attachment=True)
def multicloud():
"""
Handles the functionality on the multicloud pages.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'cloudoption' not in session:
session['cloudoption'] = constants.DEFAULT_CLOUD_OPTIONS
if 'multicloudoptions' not in session:
session['multicloudoptions'] = constants.DEFAULT_MULTICLOUD_OPTIONS
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('multicloud.html', jsonStr="", labels=labels)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = utility.generateMCJSONObj(fileManager)
session_functions.cacheCloudOption()
session_functions.cacheMultiCloudOptions()
return render_template('multicloud.html', JSONObj=JSONObj, labels=labels, loading='loading')
def hierarchy():
"""
Handles the functionality on the hierarchy page. It analyzes the various texts and
displays a dendrogram.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
# if 'dendrogramoptions' not in session: # Default settings
# session['dendrogramoptions'] = constants.DEFAULT_DENDRO_OPTIONS
labels = fileManager.getActiveLabels()
return render_template('hierarchy.html', labels=labels)
if 'dendro_download' in request.form:
# The 'Download Dendrogram' button is clicked on hierarchy.html.
# sends pdf file to downloads folder.
attachmentname = "den_"+request.form['title']+".pdf" if request.form['title'] != '' else 'dendrogram.pdf'
return send_file(pathjoin(session_functions.session_folder(),constants.RESULTS_FOLDER+"dendrogram.pdf"), attachment_filename=attachmentname, as_attachment=True)
if 'refreshThreshold' in request.form:
pdfPageNumber, score, inconsistentMax, maxclustMax, distanceMax, distanceMin, monocritMax, monocritMin, threshold = fileManager.generateDendrogram()
labels = fileManager.getActiveLabels()
return render_template('hierarchy.html', labels=labels, inconsistentMax=inconsistentMax, maxclustMax=maxclustMax, distanceMax=distanceMax, distanceMin=distanceMin, monocritMax=monocritMax, monocritMin=monocritMin, threshold=threshold)
if 'getdendro' in request.form:
#The 'Get Dendrogram' button is clicked on hierarchy.html.
pdfPageNumber, score, inconsistentMax, maxclustMax, distanceMax, distanceMin, monocritMax, monocritMin, threshold = fileManager.generateDendrogram()
session['dengenerated'] = True
labels = fileManager.getActiveLabels()
return render_template('hierarchy.html', labels=labels, pdfPageNumber=pdfPageNumber, score=score, inconsistentMax=inconsistentMax, maxclustMax=maxclustMax, distanceMax=distanceMax, distanceMin=distanceMin, monocritMax=monocritMax, monocritMin=monocritMin, threshold=threshold)
def multicloud():
"""
Handles the functionality on the multicloud pages.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'multicloudoptions' not in session:
session['multicloudoptions'] = constants.DEFAULT_MC_OPTIONS
folderPath = pathjoin(session_functions.session_folder(), constants.RESULTS_FOLDER)
if (not os.path.isdir(folderPath)):
makedirs(folderPath)
malletPath = pathjoin(folderPath, "topicFile")
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('multicloud.html', jsonStr="", labels=labels)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = fileManager.generateMCJSONObj(malletPath)
return render_template('multicloud.html', JSONObj = JSONObj, labels=labels, loading='loading')
def statistics():
"""
Handles the functionality on the Statistics page ...
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
# if len(labels) >= 1:
#FileInfoDict, corpusInfoDict = fileManager.generateStatistics()
# return render_template('statistics.html', labels=labels, FileInfoDict=FileInfoDict,
# corpusInfoDict=corpusInfoDict)
if 'analyoption' not in session:
session['analyoption'] = constants.DEFAULT_ANALIZE_OPTIONS
return render_template('statistics.html', labels=labels)
if request.method == "POST":
normalize = request.form['normalizeType']
labels = fileManager.getActiveLabels()
if len(labels) >= 1:
FileInfoDict, corpusInfoDict= utility.generateStatistics(fileManager)
session_functions.cacheAnalysisOption()
return render_template('statistics.html', labels=labels, FileInfoDict=FileInfoDict,
corpusInfoDict=corpusInfoDict, normalize=normalize)
def cut():
"""
Handles the functionality of the cut page. It cuts the files into various segments
depending on the specifications chosen by the user, and sends the text segments.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
if 'cuttingoptions' not in session:
session['cuttingoptions'] = constants.DEFAULT_CUT_OPTIONS
previews = fileManager.getPreviewsOfActive()
return render_template('cut.html', previews=previews, num_active_files=len(previews))
if 'preview' in request.form or 'apply' in request.form:
# The 'Preview Cuts' or 'Apply Cuts' button is clicked on cut.html.
session_functions.cacheCuttingOptions()
savingChanges = True if 'apply' in request.form else False # Saving changes only if apply in request form
previews = fileManager.cutFiles(savingChanges=savingChanges)
if savingChanges:
session_functions.saveFileManager(fileManager)
return render_template('cut.html', previews=previews, num_active_files=len(previews))
if 'downloadchunks' in request.form:
# The 'Download Segmented Files' button is clicked on cut.html
# sends zipped files to downloads folder
return fileManager.zipActiveFiles('cut_files.zip')
def kmeans():
"""
Handles the functionality on the kmeans page. It analyzes the various texts and
displays the class label of the files.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
defaultK = int(len(labels)/2)
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
session['kmeansdatagenerated'] = False
return render_template('kmeans.html', labels=labels, silhouettescore='', kmeansIndex=[], fileNameStr='', fileNumber=len(labels), KValue=0, defaultK=defaultK)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
session['kmeansdatagenerated'] = True
kmeansIndex, silhouetteScore, fileNameStr, KValue = fileManager.generateKMeans()
return render_template('kmeans.html', labels=labels, silhouettescore=silhouetteScore, kmeansIndex=kmeansIndex, fileNameStr=fileNameStr, fileNumber=len(labels), KValue=KValue, defaultK=defaultK)
def similarity():
"""
Handles the similarity query page functionality. Returns ranked list of files and their cosine similarities to a comparison document.
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
if 'uploadname' not in session:
session['similarities'] = constants.DEFAULT_MC_OPTIONS
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
similaritiesgenerated = False
return render_template('similarity.html', labels=labels, docsListScore="", docsListName="", similaritiesgenerated=similaritiesgenerated)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
compFile = request.form['uploadname']
docsListScore, docsListName = fileManager.generateSimilarities(compFile)
similaritiesgenerated = True
return render_template('similarity.html', labels=labels, docsListScore=docsListScore, docsListName=docsListName, similaritiesgenerated=similaritiesgenerated)
def viz():
"""
Handles the functionality on the alternate bubbleViz page with performance improvements.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'cloudoption' not in session:
session['cloudoption'] = constants.DEFAULT_CLOUD_OPTIONS
if 'bubblevisoption' not in session:
session['bubblevisoption'] = constants.DEFAULT_BUBBLEVIZ_OPTIONS
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('viz.html', JSONObj="", labels=labels)
if request.method == "POST":
# "POST" request occur when html form is submitted (i.e. 'Get Dendrogram', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = utility.generateJSONForD3(fileManager, mergedSet=True)
session_functions.cacheCloudOption()
session_functions.cacheBubbleVizOption()
return render_template('viz.html', JSONObj=JSONObj, labels=labels, loading='loading')
def downloadworkspace():
"""
Downloads workspace that stores all the session contents, which can be uploaded and restore all the workspace.
"""
fileManager = session_functions.loadFileManager()
path = fileManager.zipWorkSpace()
return send_file(path, attachment_filename=constants.WORKSPACE_FILENAME, as_attachment=True)
def rollingwindow():
"""
Handles the functionality on the rollingwindow page. It analyzes the various
texts using a rolling window of analysis.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'rwoption' not in session:
session['rwoption'] = constants.DEFAULT_ROLLINGWINDOW_OPTIONS
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
rwadatagenerated = False
# default legendlabels
legendLabels = [""]
return render_template('rwanalysis.html', labels=labels, legendLabels=legendLabels,
rwadatagenerated=rwadatagenerated)
if request.method == "POST":
# "POST" request occurs when user hits submit (Get Graph) button
labels = fileManager.getActiveLabels()
dataPoints, dataList, graphTitle, xAxisLabel, yAxisLabel, legendLabels = utility.generateRWA(fileManager)
rwadatagenerated = True
session['rwadatagenerated'] = rwadatagenerated
if 'get-RW-plot' in request.form:
# The 'Generate and Download Matrix' button is clicked on rollingwindow.html.
savePath, fileExtension = utility.generateRWmatrixPlot(dataPoints, legendLabels)
return send_file(savePath, attachment_filename="rollingwindow_matrix" + fileExtension, as_attachment=True)
if 'get-RW-data' in request.form:
# The 'Generate and Download Matrix' button is clicked on rollingwindow.html.
savePath, fileExtension = utility.generateRWmatrix(dataList)
return send_file(savePath, attachment_filename="rollingwindow_matrix" + fileExtension, as_attachment=True)
session_functions.cacheRWAnalysisOption()
if session['rwoption']['filetorollinganalyze'] == '':
session['rwoption']['filetorollinganalyze'] = unicode(labels.items()[0][0])
return render_template('rwanalysis.html', labels=labels,
data=dataPoints,
graphTitle=graphTitle,
xAxisLabel=xAxisLabel,
yAxisLabel=yAxisLabel,
legendLabels=legendLabels,
rwadatagenerated=rwadatagenerated)
def upload():
"""
Handles the functionality of the upload page. It uploads files to be used
in the current session.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
if request.method == "GET":
return render_template('upload.html', MAX_FILE_SIZE=constants.MAX_FILE_SIZE,
MAX_FILE_SIZE_INT=constants.MAX_FILE_SIZE_INT,
MAX_FILE_SIZE_UNITS=constants.MAX_FILE_SIZE_UNITS)
if 'X_FILENAME' in request.headers: # X_FILENAME is the flag to signify a file upload
# File upload through javascript
fileManager = session_functions.loadFileManager()
# --- check file name ---
fileName = request.headers[
'X_FILENAME'] # Grab the filename, which will be UTF-8 percent-encoded (e.g. '%E7' instead of python's '\xe7')
if isinstance(fileName, unicode): # If the filename comes through as unicode
fileName = fileName.encode('ascii') # Convert to an ascii string
fileName = unquote(fileName).decode(
'utf-8') # Unquote using urllib's percent-encoding decoder (turns '%E7' into '\xe7'), then deocde it
# --- end check file name ---
if fileName.endswith('.lexos'):
fileManager.handleUploadWorkSpace()
# update filemanager
fileManager = session_functions.loadFileManager()
fileManager.updateWorkspace()
else:
fileManager.addUploadFile(request.data, fileName)
session_functions.saveFileManager(fileManager)
return 'success'
def kmeans():
"""
Handles the functionality on the kmeans page. It analyzes the various texts and
displays the class label of the files.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
defaultK = int(len(labels) / 2)
if 'analyoption' not in session:
session['analyoption'] = constants.DEFAULT_ANALIZE_OPTIONS
if 'kmeanoption' not in session:
session['kmeanoption'] = constants.DEFAULT_KMEAN_OPTIONS
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
return render_template('kmeans.html', labels=labels, silhouettescore='', kmeansIndex=[], fileNameStr='',
fileNumber=len(labels), KValue=0, defaultK=defaultK,
colorChartStr='', kmeansdatagenerated=False)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
if request.form['viz'] == 'PCA':
kmeansIndex, silhouetteScore, fileNameStr, KValue, colorChartStr = utility.generateKMeansPCA(fileManager)
session_functions.cacheAnalysisOption()
session_functions.cacheKmeanOption()
session_functions.saveFileManager(fileManager)
return render_template('kmeans.html', labels=labels, silhouettescore=silhouetteScore,
kmeansIndex=kmeansIndex,
fileNameStr=fileNameStr, fileNumber=len(labels), KValue=KValue, defaultK=defaultK,
colorChartStr=colorChartStr, kmeansdatagenerated=True)
elif request.form['viz'] == 'Voronoi':
kmeansIndex, silhouetteScore, fileNameStr, KValue, colorChartStr, finalPointsList, finalCentroidsList, textData, maxVal = utility.generateKMeansVoronoi(fileManager)
session_functions.cacheAnalysisOption()
session_functions.cacheKmeanOption()
session_functions.saveFileManager(fileManager)
return render_template('kmeans.html', labels=labels, silhouettescore=silhouetteScore,
kmeansIndex=kmeansIndex, fileNameStr=fileNameStr, fileNumber=len(labels),
KValue=KValue, defaultK=defaultK, colorChartStr=colorChartStr,
finalPointsList=finalPointsList, finalCentroidsList=finalCentroidsList,
textData=textData, maxVal=maxVal, kmeansdatagenerated=True)
def rollingwindow():
"""
Handles the functionality on the rollingwindow page. It analyzes the various
texts using a rolling window of analysis.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
#"GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
session['rwadatagenerated'] = False
#default legendlabels
legendLabels = [""]
return render_template('rwanalysis.html', labels=labels, legendLabels=legendLabels)
if request.method == "POST":
#"POST" request occurs when user hits submit (Get Graph) button
labels = fileManager.getActiveLabels()
dataPoints, dataList, graphTitle, xAxisLabel, yAxisLabel, legendLabels = fileManager.generateRWA()
session['rwadatagenerated'] = True
if 'get-RW-plot' in request.form:
#The 'Generate and Download Matrix' button is clicked on csvgenerator.html.
savePath, fileExtension = fileManager.generateRWmatrixPlot(dataPoints, legendLabels)
return send_file(savePath, attachment_filename="rollingwindow_matrix"+fileExtension, as_attachment=True)
if 'get-RW-data' in request.form:
#The 'Generate and Download Matrix' button is clicked on csvgenerator.html.
savePath, fileExtension = fileManager.generateRWmatrix(dataList)
return send_file(savePath, attachment_filename="rollingwindow_matrix"+fileExtension, as_attachment=True)
return render_template('rwanalysis.html', labels=labels,
data=dataPoints,
graphTitle=graphTitle,
xAxisLabel=xAxisLabel,
yAxisLabel=yAxisLabel,
legendLabels=legendLabels)
def select():
"""
Handles the functionality of the select page. Its primary role is to activate/deactivate
specific files depending on the user's input.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager() # Usual loading of the FileManager
if request.method == "GET":
activePreviews = fileManager.getPreviewsOfActive()
inactivePreviews = fileManager.getPreviewsOfInactive()
return render_template('select.html', activeFiles=activePreviews, inactiveFiles=inactivePreviews)
if 'toggleFile' in request.headers:
# Catch-all for any POST request.
# On the select page, POSTs come from JavaScript AJAX XHRequests.
fileID = int(request.data)
fileManager.toggleFile(fileID) # Toggle the file from active to inactive or vice versa
elif 'setLabel' in request.headers:
newLabel = (request.headers['setLabel']).decode('utf-8')
fileID = int(request.data)
fileManager.files[fileID].label = newLabel
elif 'disableAll' in request.headers:
fileManager.disableAll()
elif 'selectAll' in request.headers:
fileManager.enableAll()
elif 'applyClassLabel' in request.headers:
fileManager.classifyActiveFiles()
elif 'deleteActive' in request.headers:
fileManager.deleteActiveFiles()
session_functions.saveFileManager(fileManager)
return '' # Return an empty string because you have to return something
def clustering():
"""
Menu page for clustering. Let's you select either hierarchical or kmeans clustering page.
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
return render_template('clustering.html', labels=labels)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
return render_template('clustering.html', labels=labels)
def tokenizer():
"""
Handles the functionality on the tokenize page. It analyzes the texts to produce
and send various frequency matrices.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
if 'csvoptions' not in session:
session['csvoptions'] = constants.DEFAULT_CSV_OPTIONS
labels = fileManager.getActiveLabels()
return render_template('tokenizer.html', labels=labels)
if 'gen-csv' in request.form:
#The 'Generate and Visualize Matrix' button is clicked on tokenizer.html.
DocTermSparseMatrix, countMatrix = fileManager.generateCSVMatrix(roundDecimal=True)
countMatrix = zip(*countMatrix)
dtm = []
for row in xrange(1,len(countMatrix)):
dtm.append(list(countMatrix[row]))
matrixTitle = list(countMatrix[0])
matrixTitle[0] = "Token"
matrixTitle[0] = matrixTitle[0].encode("utf-8")
labels = fileManager.getActiveLabels()
session_functions.saveFileManager(fileManager)
return render_template('tokenizer.html', labels=labels, matrixData=dtm, matrixTitle=matrixTitle, matrixExist=True)
if 'get-csv' in request.form:
#The 'Download Matrix' button is clicked on tokenizer.html.
session_functions.cacheCSVOptions()
savePath, fileExtension = fileManager.generateCSV()
session_functions.saveFileManager(fileManager)
return send_file(savePath, attachment_filename="frequency_matrix"+fileExtension, as_attachment=True)
def viz():
"""
Handles the functionality on the alternate bubbleViz page with performance improvements.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('viz.html', JSONObj="", labels=labels)
if request.method == "POST":
# "POST" request occur when html form is submitted (i.e. 'Get Dendrogram', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = fileManager.generateJSONForD3(mergedSet=True)
return render_template('viz.html', JSONObj=JSONObj, labels=labels, loading='loading')
def topword():
"""
Handles the topword page functionality. Returns ranked list of topwords
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
if 'uploadname' not in session:
session['topword'] = constants.DEFAULT_MC_OPTIONS
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
return render_template('topword.html', labels=labels, docsListScore="", docsListName="", topwordsgenerated=False)
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
inputFiles = request.form['chunkgroups']
docsListScore, docsListName = fileManager.generateSimilarities(inputFiles)
return render_template('topword.html', labels=labels, docsListScore=docsListScore, docsListName=docsListName, topwordsgenerated=True)
def upload():
"""
Handles the functionality of the upload page. It uploads files to be used
in the current session.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
if request.method == "GET":
return render_template('upload.html')
if 'X_FILENAME' in request.headers: # X_FILENAME is the flag to signify a file upload
# File upload through javascript
fileManager = session_functions.loadFileManager()
fileName = request.headers['X_FILENAME'] # Grab the filename, which will be UTF-8 percent-encoded (e.g. '%E7' instead of python's '\xe7')
if isinstance(fileName, unicode): # If the filename comes through as unicode
fileName = fileName.encode('ascii') # Convert to an ascii string
fileName = unquote(fileName).decode('utf-8') # Unquote using urllib's percent-encoding decoder (turns '%E7' into '\xe7'), then deocde it
# detect (and apply) the encoding type of the file's contents
# since chardet runs slow, initially detect (only) first 500 chars;
# if that fails, chardet entire file for a fuller test
try:
encodingDetect = chardet.detect(request.data[:500]) # Detect the encoding from the first 500 characters
encodingType = encodingDetect['encoding']
fileString = request.data.decode(encodingType) # Grab the file contents, which were encoded/decoded automatically into python's format
except:
encodingDetect = chardet.detect(request.data) # :( ... ok, detect the encoding from entire file
encodingType = encodingDetect['encoding']
fileString = request.data.decode(encodingType) # Grab the file contents, which were encoded/decoded automatically into python's format
fileManager.addFile(fileName, fileString) # Add the file to the FileManager
session_functions.saveFileManager(fileManager)
return 'success'
def scrub():
"""
Handles the functionality of the scrub page. It scrubs the files depending on the
specifications chosen by the user, with an option to download the scrubbed files.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
if 'scrubbingoptions' not in session:
session['scrubbingoptions'] = constants.DEFAULT_SCRUB_OPTIONS
previews = fileManager.getPreviewsOfActive()
tagsPresent, DOEPresent = fileManager.checkActivesTags()
return render_template('scrub.html', previews=previews, haveTags=tagsPresent, haveDOE=DOEPresent)
if 'preview' in request.form or 'apply' in request.form:
#The 'Preview Scrubbing' or 'Apply Scrubbing' button is clicked on scrub.html.
session_functions.cacheAlterationFiles()
session_functions.cacheScrubOptions()
# saves changes only if 'Apply Scrubbing' button is clicked
savingChanges = True if 'apply' in request.form else False
previews = fileManager.scrubFiles(savingChanges=savingChanges)
tagsPresent, DOEPresent = fileManager.checkActivesTags()
if savingChanges:
session_functions.saveFileManager(fileManager)
return render_template('scrub.html', previews=previews, haveTags=tagsPresent, haveDOE=DOEPresent)
if 'download' in request.form:
# The 'Download Scrubbed Files' button is clicked on scrub.html.
# sends zipped files to downloads folder.
return fileManager.zipActiveFiles('scrubbed.zip')
def wordcloud():
"""
Handles the functionality on the visualisation page -- a prototype for displaying
single word cloud graphs.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('wordcloud.html', labels=labels)
if request.method == "POST":
# "POST" request occur when html form is submitted (i.e. 'Get Dendrogram', 'Download...')
labels = fileManager.getActiveLabels()
JSONObj = fileManager.generateJSONForD3(mergedSet=True)
return render_template('wordcloud.html', labels=labels, JSONObj=JSONObj)
def tokenizer():
"""
Handles the functionality on the tokenizer page. It analyzes the texts to produce
and send various frequency matrices.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
if 'analyoption' not in session:
session['analyoption'] = constants.DEFAULT_ANALIZE_OPTIONS
if 'csvoptions' not in session:
session['csvoptions'] = constants.DEFAULT_CSV_OPTIONS
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
return render_template('tokenizer.html', labels=labels, matrixExist=False)
if 'gen-csv' in request.form:
# The 'Generate and Visualize Matrix' button is clicked on tokenizer.html.
session_functions.cacheAnalysisOption()
session_functions.cacheCSVOptions()
labels = fileManager.getActiveLabels()
matrixTitle, tableStr = utility.generateTokenizeResults(fileManager)
session_functions.saveFileManager(fileManager)
return render_template('tokenizer.html', labels=labels, matrixTitle=matrixTitle,
tableStr=tableStr, matrixExist=True)
if 'get-csv' in request.form:
# The 'Download Matrix' button is clicked on tokenizer.html.
session_functions.cacheAnalysisOption()
session_functions.cacheCSVOptions()
savePath, fileExtension = utility.generateCSV(fileManager)
session_functions.saveFileManager(fileManager)
return send_file(savePath, attachment_filename="frequency_matrix" + fileExtension, as_attachment=True)
def hierarchy():
"""
Handles the functionality on the hierarchy page. It analyzes the various texts and
displays a dendrogram.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager()
leq = '≤'.decode('utf-8')
if 'analyoption' not in session:
session['analyoption'] = constants.DEFAULT_ANALIZE_OPTIONS
if 'hierarchyoption' not in session:
session['hierarchyoption'] = constants.DEFAULT_HIERARCHICAL_OPTIONS
if request.method == "GET":
# "GET" request occurs when the page is first loaded.
labels = fileManager.getActiveLabels()
thresholdOps = {}
return render_template('hierarchy.html', labels=labels, thresholdOps=thresholdOps)
if 'dendro_download' in request.form:
# The 'Download Dendrogram' button is clicked on hierarchy.html.
# sends pdf file to downloads folder.
utility.generateDendrogram(fileManager)
attachmentname = "den_" + request.form['title'] + ".pdf" if request.form['title'] != '' else 'dendrogram.pdf'
session_functions.cacheAnalysisOption()
session_functions.cacheHierarchyOption()
return send_file(pathjoin(session_functions.session_folder(), constants.RESULTS_FOLDER + "dendrogram.pdf"),
attachment_filename=attachmentname, as_attachment=True)
if 'dendroSVG_download' in request.form:
utility.generateDendrogram(fileManager)
attachmentname = "den_" + request.form['title'] + ".svg" if request.form['title'] != '' else 'dendrogram.svg'
session_functions.cacheAnalysisOption()
session_functions.cacheHierarchyOption()
return send_file(pathjoin(session_functions.session_folder(), constants.RESULTS_FOLDER + "dendrogram.svg"),
attachment_filename=attachmentname, as_attachment=True)
if 'getdendro' in request.form:
# The 'Get Dendrogram' button is clicked on hierarchy.html.
pdfPageNumber, score, inconsistentMax, maxclustMax, distanceMax, distanceMin, monocritMax, monocritMin, threshold = utility.generateDendrogram(fileManager)
session['dengenerated'] = True
labels = fileManager.getActiveLabels()
inconsistentOp = "0 " + leq + " t " + leq + " " + str(inconsistentMax)
maxclustOp = "2 " + leq + " t " + leq + " " + str(maxclustMax)
distanceOp = str(distanceMin) + " " + leq + " t " + leq + " " + str(distanceMax)
monocritOp = str(monocritMin) + " " + leq + " t " + leq + " " + str(monocritMax)
thresholdOps = {"inconsistent": inconsistentOp, "maxclust": maxclustOp, "distance": distanceOp,
"monocrit": monocritOp}
session_functions.saveFileManager(fileManager)
session_functions.cacheAnalysisOption()
session_functions.cacheHierarchyOption()
return render_template('hierarchy.html', labels=labels, pdfPageNumber=pdfPageNumber, score=score,
inconsistentMax=inconsistentMax, maxclustMax=maxclustMax, distanceMax=distanceMax,
distanceMin=distanceMin, monocritMax=monocritMax, monocritMin=monocritMin,
threshold=threshold, thresholdOps=thresholdOps)
def topword():
"""
Handles the topword page functionality.
"""
fileManager = session_functions.loadFileManager()
labels = fileManager.getActiveLabels()
if 'topwordoption' not in session:
session['topwordoption'] = constants.DEFAULT_TOPWORD_OPTIONS
if 'analyoption' not in session:
session['analyoption'] = constants.DEFAULT_ANALIZE_OPTIONS
if request.method == 'GET':
# 'GET' request occurs when the page is first loaded
# get the class label and eliminate the id (this is not the unique id in filemanager)
ClassdivisionMap = fileManager.getClassDivisionMap()[1:]
# if there is no file active (ClassdivisionMap == []) just jump to the page
# notice python eval from right to left
# if there is only one chunk then make the default test prop-z for all
if ClassdivisionMap != [] and len(ClassdivisionMap[0]) == 1:
session['topwordoption']['testMethodType'] = 'pz'
session['topwordoption']['testInput'] = 'useAll'
return render_template('topword2.html', labels=labels, classmap=ClassdivisionMap, topwordsgenerated='class_div')
if request.method == "POST":
# 'POST' request occur when html form is submitted (i.e. 'Get Graphs', 'Download...')
if request.form['testMethodType'] == 'pz':
if request.form['testInput'] == 'useclass':
result = utility.GenerateZTestTopWord(fileManager)
# only give the user a preview of the topWord
for key in result.keys():
if len(result[key]) > 20:
result.update({key: result[key][:20]})
session_functions.cacheAnalysisOption()
session_functions.cacheTopwordOptions()
return render_template('topword2.html', result=result, labels=labels, topwordsgenerated='pz_class')
else:
result = utility.GenerateZTestTopWord(fileManager)
print result[0]
# only give the user a preview of the topWord
for i in range(len(result)):
if len(result[i][1]) > 20:
result[i][1] = result[i][1][:20]
print result[1]
session_functions.cacheAnalysisOption()
session_functions.cacheTopwordOptions()
return render_template('topword2.html', result=result, labels=labels, topwordsgenerated='pz_all')
else:
result = utility.generateKWTopwords(fileManager)
print result
# only give the user a preview of the topWord
if len(result) > 50:
result = result[:50]
session_functions.cacheAnalysisOption()
session_functions.cacheTopwordOptions()
return render_template('topword2.html', result=result, labels=labels, topwordsgenerated='KW')
def select():
"""
Handles the functionality of the select page. Its primary role is to activate/deactivate
specific files depending on the user's input.
Note: Returns a response object (often a render_template call) to flask and eventually
to the browser.
"""
fileManager = session_functions.loadFileManager() # Usual loading of the FileManager
if request.method == "GET":
rows = fileManager.getPreviewsOfAll()
for row in rows:
if row["state"] == True:
row["state"] = "DTTT_selected selected"
else:
row["state"] = ""
return render_template('select.html', rows=rows)
if 'previewTest' in request.headers:
fileID = int(request.data)
fileLabel = fileManager.files[fileID].label
filePreview = fileManager.files[fileID].getPreview()
previewVals = {"id": fileID, "label": fileLabel, "previewText": filePreview}
import json
return json.dumps(previewVals)
if 'toggleFile' in request.headers:
# Catch-all for any POST request.
# On the select page, POSTs come from JavaScript AJAX XHRequests.
fileID = int(request.data)
fileManager.toggleFile(fileID) # Toggle the file from active to inactive or vice versa
elif 'setLabel' in request.headers:
newName = (request.headers['setLabel']).decode('utf-8')
fileID = int(request.data)
fileManager.files[fileID].setName(newName)
fileManager.files[fileID].label = newName
elif 'setClass' in request.headers:
newClassLabel = (request.headers['setClass']).decode('utf-8')
fileID = int(request.data)
fileManager.files[fileID].setClassLabel(newClassLabel)
elif 'disableAll' in request.headers:
fileManager.disableAll()
elif 'selectAll' in request.headers:
fileManager.enableAll()
elif 'applyClassLabel' in request.headers:
fileManager.classifyActiveFiles()
elif 'deleteActive' in request.headers:
fileManager.deleteActiveFiles()
elif 'deleteRow' in request.headers:
fileManager.deleteFiles(request.form.keys()) # delete the file in request.form
session_functions.saveFileManager(fileManager)
return '' # Return an empty string because you have to return something