def write_raw_to_html_csv(user_raw_data, cat_parser):
"""
:param cat_parser: cat parser instance with cat values
:param user_raw_data: data['username'][username,duration,requests]
:return: creates csv and html files in timestamped directory
"""
user_data = "["
for user in user_raw_data:
#user_data.append("{'username': '******', 'duration': %s, 'requests': %s},"
user_data += (
"{'username': '******', 'duration': %s, 'requests': %s}," %
(user_raw_data[user]["user"], user_raw_data[user]["duration"],
user_raw_data[user]["requests"]))
user_data += "]"
cat_data = "["
for cat in cat_parser.result:
cat_data += (
"{'category': '%s', "
"'duration': %s, "
"'size': %s, "
"'requests': %s}," %
(cat, cat_parser.result[cat]["duration"],
cat_parser.result[cat]["size"], cat_parser.result[cat]["count"]))
cat_data += "]"
user_builder = HtmlBuilder.HtmlBuilder(user_data)
cat_builder = CatBuild.CategoryHtmlBuilder(cat_data,
user_builder.target_dir)
# why does this work? dir_name is only the subdir!
write_csv(user_raw_data, user_builder.dir_name, "user.csv")
write_cat_csv(cat_parser.result, user_builder.dir_name, "category.csv")
def write_raw_to_html_csv(user_raw_data):
"""
:param user_raw_data: data['username'][username,duration,requests]
:return: creates csv and html files in timestamped directory
"""
user_data = "["
for user in user_raw_data:
#user_data.append("{'username': '******', 'duration': %s, 'requests': %s},"
user_data += (
"{'username': '******', 'duration': %s, 'requests': %s}," %
(user_raw_data[user]["user"], user_raw_data[user]["duration"],
user_raw_data[user]["requests"]))
user_data += "]"
builder = hb.HtmlBuilder(user_data)
write_csv(user_raw_data, builder.dir_name)
# 获取文件列表，写入html
import os
import HtmlBuilder
root_path = os.path.curdir
html_builder = HtmlBuilder.HtmlBuilder('dir.html')
def generate_html(path):
os.chdir(path)
for file in os.listdir(os.curdir):
# file = os.path.join(path, file)
if os.path.isdir(file):
html_builder.build_tab(0)
html_builder.build_path(file)
generate_html(os.path.abspath(file))
os.chdir(os.pardir)
elif os.path.isfile(file):
html_builder.build_tab(1)
html_builder.build_file(file)
html_builder.build_h1('Directory')
generate_html(root_path)
html_builder.finish()
def makeSentenceListPage(self):
#print >> sys.stderr, "Making sentence page"
builder = HtmlBuilder()
builder.newPage("Sentences","")
builder.header("Sentences")
builder.table(1,True)
builder.tableHead()
builder.tableRow()
builder.tableHeader("id",True)
builder.tableHeader("text",True)
builder.tableHeader("origId",True)
#builder.tableHeader("set",True)
builder.tableHeader("entities",True)
builder.tableHeader("edges",True)
#builder.tableHeader("examples",True)
#builder.tableHeader("classifications",True)
##builder.tableHeader("pairs",True)
##builder.tableHeader("int",True)
builder.tableHeader("tp ",True)
builder.tableHeader("fp ",True)
builder.tableHeader("fn ",True)
builder.closeElement() # close tableRow
builder.closeElement() # close tableHead
builder.tableBody()
count = 0
for sentence in self.sentences:
count += 1
if sentence == None:
continue
#sentence = sentencesById[key]
sentenceId = sentence[0].getSentenceId()
builder.tableRow()
builder.tableData()
builder.link("sentences/" + sentenceId + "-" + str(count) + ".html", sentenceId)
builder.closeElement()
text = sentence[0].sentenceElement.attrib["text"]
if len(text) > 80:
text = text[:80] + "..."
builder.tableData(text,True)
builder.tableData(self.__getOrigId(sentence[0].sentenceElement),True)
#if hasattr(sentence[0], "visualizationSet"):
# builder.tableData(str(sentence[0].visualizationSet),True)
#else:
# builder.tableData("N/A",True)
#builder.tableData(str(sentence[1]),True)
#builder.tableData(str(sentence[2]),True)
builder.tableData(str(sentence[0].stats["entities"]),True)
builder.tableData(str(sentence[0].stats["edges"]),True)
builder.tableData(str(sentence[0].stats["tp"]),True)
builder.tableData(str(sentence[0].stats["fp"]),True)
builder.tableData(str(sentence[0].stats["fn"]),True)
#builder.tableData(str(len(sentence.annotationDependencies)),True)
#builder.tableData(str(len(sentence.entities)),True)
#pairs = sentence.pairs
#builder.tableData(str(len(pairs)),True)
#numInteractions = 0
#for pair in pairs:
# if pair.get("interaction") == "True":
# numInteractions += 1
#builder.tableData(str(numInteractions),True)
#tp = sentence[3]
#fp = sentence[4]
#builder.tableData(str(tp),True)
#builder.tableData(str(fp),True)
builder.closeElement() # close tableRow
builder.closeElement() # close tableBody
builder.closeElement() # close table
builder.write(self.outDir+"/sentences.html")
def makeSentencePage(self, sentenceGraph, examples, classificationsByExample, prevAndNextId=None, goldGraph=None):
# Store info for sentence list
sentenceId = sentenceGraph.getSentenceId()
self.sentences.append([sentenceGraph,0,0,0,0])
sentenceIndex = len(self.sentences)
sentenceGraph.stats = {"entities":0,"edges":0,"tp":0,"fp":0,"tn":0,"fn":0}
visualizationSet = None
if examples != None:
for example in examples:
if visualizationSet == None:
visualizationSet = example[3]["visualizationSet"]
else:
assert(visualizationSet == example[3]["visualizationSet"])
self.sentences[-1][1] += 1
if classificationsByExample.has_key(example[0]):
classification = classificationsByExample[example[0]]
self.sentences[-1][2] += 1
if classification[1] == "tp":
self.sentences[-1][3] += 1
elif classification[1] == "fp":
self.sentences[-1][4] += 1
sentenceGraph.visualizationSet = visualizationSet
# Make the page
entityElements = sentenceGraph.entities
entityTextById = {}
for entityElement in entityElements:
entityTextById[entityElement.get("id")] = entityElement.get("text")
# Boot-it NG
builder = HtmlBuilder()
builder.newPage("Sentence " + sentenceId, "../")
builder.addScript("../js/highlight_svg.js")
builder.body.set("onload","for(i in document.forms){document.forms[i].reset();}")
builder.div()
builder.header("Sentence " + sentenceId,1)
#builder.lineBreak()
if prevAndNextId != None:
if prevAndNextId[0] != None:
builder.link(prevAndNextId[0]+"-"+str(sentenceIndex-1)+".html","previous")
else:
builder.span("previous","color:#0000FF;")
if prevAndNextId[1] != None:
builder.link(prevAndNextId[1]+"-"+str(sentenceIndex+1)+".html","next")
else:
builder.span("next","color:#0000FF;")
builder.span("Original ID: " + self.__getOrigId(sentenceGraph.sentenceElement))
builder.span("Index: " + str(sentenceIndex))
builder.closeElement() # div
builder.lineBreak()
# Parse SVG
builder.header("Parse",4)
svgTokens = GraphToSVG.tokensToSVG(sentenceGraph.tokens, True)
nxDepGraph = NX10.MultiDiGraph()
for edge in sentenceGraph.dependencyGraph.edges:
nxDepGraph.add_edge(edge[0], edge[1], element=edge[2])
svgDependencies = GraphToSVG.edgesToSVG(svgTokens, nxDepGraph)
svgElement = GraphToSVG.writeSVG(svgTokens, svgDependencies,self.outDir+"/svg/"+sentenceId+"-"+str(sentenceIndex)+".svg")
builder.svg("../svg/" + sentenceId + "-"+str(sentenceIndex)+".svg",svgElement.attrib["width"],svgElement.attrib["height"],id="dep_graph")
builder.lineBreak()
# Annotation SVG
builder.header("Annotation",4)
if goldGraph != None:
# Check for named entities
isNameByToken = {}
for token in goldGraph.tokens:
if goldGraph.getTokenText(token) == "NAMED_ENT":
isNameByToken[token] = True
else:
isNameByToken[token] = False
#arcStyles, labelStyles = self.getMatchingEdgeStyles(goldGraph.interactionGraph, goldGraph.dependencyGraph, "orange", "#F660AB" )
svgTokens = GraphToSVG.tokensToSVG(goldGraph.tokens, False, goldGraph.entitiesByToken, None, isNameByToken)
nxGraph = NX10.MultiDiGraph()
for edge in goldGraph.interactionGraph.edges:
nxGraph.add_edge(edge[0], edge[1], element=edge[2])
svgInteractionEdges = GraphToSVG.edgesToSVG(svgTokens, nxGraph)
svgElement = GraphToSVG.writeSVG(svgTokens, svgInteractionEdges,self.outDir+"/svg/"+sentenceId+"-"+str(sentenceIndex)+"_ann.svg")
elif sentenceGraph.interactionGraph != None:
# Check for named entities
isNameByToken = {}
for token in sentenceGraph.tokens:
if sentenceGraph.getTokenText(token) == "NAMED_ENT":
isNameByToken[token] = True
else:
isNameByToken[token] = False
#arcStyles, labelStyles = self.getMatchingEdgeStyles(sentenceGraph.interactionGraph, sentenceGraph.dependencyGraph, "orange", "#F660AB" )
svgTokens = GraphToSVG.tokensToSVG(sentenceGraph.tokens, False, sentenceGraph.entitiesByToken, None, isNameByToken)
nxGraph = NX10.MultiDiGraph()
for edge in goldGraph.interactionGraph.edges:
nxGraph.add_edge(edge[0], edge[1], element=edge[2])
svgInteractionEdges = GraphToSVG.edgesToSVG(svgTokens, nxGraph)
svgElement = GraphToSVG.writeSVG(svgTokens, svgInteractionEdges,self.outDir+"/svg/"+sentenceId + "-"+str(sentenceIndex)+"_ann.svg")
builder.svg("../svg/" + sentenceId + "-"+str(sentenceIndex)+"_ann.svg",svgElement.attrib["width"],svgElement.attrib["height"],id="ann_graph")
builder.lineBreak()
# Classification svg
if classificationsByExample != None:
self.makeExampleGraph(builder, sentenceGraph, examples, classificationsByExample, sentenceIndex)
elif goldGraph != None:
sentenceGraph.stats = self.makeExampleGraphWithGold(builder, sentenceGraph, goldGraph, sentenceIndex)
builder.table(0,align="center",width="100%")
builder.tableRow()
# interactions
pairElements = sentenceGraph.interactions
builder.tableData(valign="top")
builder.header("Interactions",4)
builder.table(1,True)
builder.tableHead()
builder.tableRow()
builder.tableHeader("id", True)
builder.tableHeader("type", True)
builder.tableHeader("e1", True)
builder.tableHeader("e2", True)
builder.tableHeader("e1 word", True)
builder.tableHeader("e2 word", True)
#builder.tableHeader("interaction", True)
#th = builder.tableHeader("view",True)
#th.set("class","{sorter: false}")
builder.closeElement()
builder.closeElement() # close tableHead
builder.tableBody()
for pairElement in sentenceGraph.interactions:
tr = builder.tableRow()
#tr.set( "onmouseover", getPairHighlightCommand("main_parse",pairElement.get("e1"),pairElement.get("e2"),entityTokens,"highlightPair") )
#tr.set( "onmouseout", getPairHighlightCommand("main_parse",pairElement.get("e1"),pairElement.get("e2"),entityTokens,"deHighlightPair") )
builder.tableData(pairElement.get("id").split(".")[-1][1:], True)
builder.tableData(pairElement.get("type"), True)
builder.tableData(pairElement.get("e1").split(".")[-1][1:], True)
builder.tableData(pairElement.get("e2").split(".")[-1][1:], True)
builder.tableData(entityTextById[pairElement.get("e1")], True)
builder.tableData(entityTextById[pairElement.get("e2")], True)
#builder.tableData("Dummy", True)
#builder.tableData()
#builder.form()
#input = builder.formInput("checkbox")
##input.set("onClick",getPairHighlightCommand("main_parse",pairElement.get("e1"),pairElement.get("e2"),entityTokens,"toggleSelectPair",pairElement.get("id")) )
#builder.closeElement() # form
#builder.closeElement() # tableData
builder.closeElement()
builder.closeElement() # close tableBody
builder.closeElement() # close table
# entities
builder.tableData(valign="top")
builder.header("Entities",4)
builder.table(1,True)
builder.tableHead()
builder.tableRow()
builder.tableHeader("id", True)
builder.tableHeader("text", True)
builder.tableHeader("type", True)
builder.tableHeader("charOffset", True)
builder.closeElement() # close tableRow
builder.closeElement() # close tableHead
entityElements = sentenceGraph.entities
builder.tableBody()
for entityElement in entityElements:
builder.tableRow()
builder.tableData(entityElement.get("id").split(".")[-1][1:], True)
builder.tableData(entityElement.get("text"), True)
if entityElement.attrib["isName"] == "True":
builder.tableData("["+entityElement.get("type")+"]", True)
else:
builder.tableData(entityElement.get("type"), True)
charOffset = entityElement.get("charOffset")
charOffsetSplits = charOffset.split(",")
headOffset = entityElement.get("headOffset")
charOffset = ""
headFound = False
for charOffsetSplit in charOffsetSplits:
if charOffset != "":
charOffset += ","
if charOffsetSplit == headOffset:
charOffset += "<u>" + charOffsetSplit + "</u>"
headFound = True
else:
charOffset += charOffsetSplit
if not headFound:
charOffset += " (<u>" + headOffset + "</u>)"
builder.tableData(charOffset, True)
builder.closeElement()
builder.closeElement() # close tableBody
builder.closeElement() # close table
builder.closeElement() # close row
builder.closeElement() # close table
builder.closeElement() # close row
builder.closeElement() # close table
# Examples
if examples != None:
builder.header("Examples",4)
for example in examples:
string = example[0]
if classificationsByExample.has_key(example[0]):
string += " (" + classificationsByExample[example[0]][1] + ")"
string += ":"
features = example[2]
if self.featureSet != None:
featureNames = []
for key in features.keys():
featureNames.append(self.featureSet.getName(key))
featureNames.sort()
for featureName in featureNames:
string += " " + featureName + ":" + str(features[self.featureSet.getId(featureName)])
else:
keys = features.keys()
keys.sort()
for key in keys:
featureName = str(key)
string += " " + featureName + ":" + str(features[key])
#string += "\n"
builder.span(string)
builder.lineBreak()
builder.lineBreak()
builder.write(self.outDir + "/sentences/"+sentenceId+"-"+str(sentenceIndex)+".html")
repairApostrophes(self.outDir + "/sentences/"+sentenceId+"-"+str(sentenceIndex)+".html")
