def __init__(self, _graph, _descriptorListFile, _clusterProperty):
starttime = datetime.datetime.now()
self.layoutAlgorithm = "GEM (Frick)"
if _descriptorListFile != "":
self.fullTypeList = self.readTypeList(_descriptorListFile)
else:
self.fullTypeList = set()
descP = _graph.getStringProperty("descripteurs")
for n in _graph.getNodes():
self.fullTypeList.update(descP.getNodeValue(n).split(';'))
self.fullTypeList = list(self.fullTypeList)
self.analyseGraph = tlp.newSubGraph(_graph.getSuperGraph(), _graph.getName()+" Analysis")
#self.docGraph = tlp.newSubGraph(self.analyseGraph, "documents")
self.clusterGraph = tlp.newSubGraph(self.analyseGraph, "clusters")
self.detailGraph = tlp.newSubGraph(self.analyseGraph, "details")
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution1: ",currentTime
#tlp.copyToGraph(self.docGraph, _graph)
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution1.1: ",currentTime
clusterProperty = _graph.getDoubleProperty(_clusterProperty)
minP = clusterProperty.getNodeMin(_graph)
maxP = clusterProperty.getNodeMax(_graph) +1
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution1.2: ",currentTime
self.clusterToNodes = {}
for c in range(int(minP), int(maxP)):
self.clusterToNodes[c] = []
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution1.3: ",currentTime
for n in _graph.getNodes():
cluster = clusterProperty.getNodeValue(n)
self.clusterToNodes[int(cluster)].append(n)
#print clusterToNodes
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution1.4: ",currentTime
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution2: ",currentTime
self.clusterToClusterNode = {} #nodes in the cluster subgraph corresponding to the cluster
self.clusterToSubGraph = {} #subgraph in the detail subgraph corresponding to the cluster
self.clusterToTypeList = {}
clusterCount = 0
for c in self.clusterToNodes.keys():
#if c >1:
# break
print "handling cluster ",clusterCount
clusterCount = clusterCount+1
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution: ",currentTime
clustercount= clusterCount+1
self.clusterToClusterNode[c] = self.clusterGraph.addNode()
cGraph =_graph.inducedSubGraph(self.clusterToNodes[c])
# builds the subgraph hierarchy
self.clusterToSubGraph[c] = [tlp.newSubGraph(self.detailGraph, "%d"%c)]
self.clusterToSubGraph[c].append(tlp.newSubGraph(self.clusterToSubGraph[c][0], "Documents"))
self.clusterToSubGraph[c].append(tlp.newSubGraph(self.clusterToSubGraph[c][0], "Descriptors"))
self.copyToGraph(self.clusterToSubGraph[c][1], cGraph)
#analyse the graph
cAnalysis = clusterAnalysisLgt(cGraph, self.fullTypeList)
cAnalysis.typeGraph.computeLayoutProperty(self.layoutAlgorithm,cAnalysis.typeGraph.getLayoutProperty("viewLayout"))
tlp.copyToGraph(self.clusterToSubGraph[c][2], cAnalysis.typeGraph)
self.clusterToTypeList[c] = cAnalysis.typeList
title = " ".join(cAnalysis.typeList)
self.clusterToSubGraph[c][0].setName(title)
self.clusterToSubGraph[c][0].computeLayoutProperty("Connected Component Packing", self.clusterToSubGraph[c][0].getLayoutProperty("viewLayout"))
self.clusterGraph.getStringProperty("viewLabel").setNodeValue(self.clusterToClusterNode[c], title)
self.clusterGraph.getStringProperty("clusterName").setNodeValue(self.clusterToClusterNode[c], title)
self.clusterGraph.getStringProperty("clusterTypeList").setNodeValue(self.clusterToClusterNode[c], title)
self.clusterGraph.getDoubleProperty("clusterCoherence").setNodeValue(self.clusterToClusterNode[c], cAnalysis.globalCoherence)
self.clusterGraph.getDoubleProperty("clusterGraphID").setNodeValue(self.clusterToClusterNode[c], self.clusterToSubGraph[c][0].getId())
self.clusterGraph.getDoubleProperty("clusterCosine").setNodeValue(self.clusterToClusterNode[c], cAnalysis.globalCosine)
self.clusterGraph.getDoubleProperty("dateBegin").setNodeValue(self.clusterToClusterNode[c], cAnalysis.dateBegin)
self.clusterGraph.getDoubleProperty("dateEnd").setNodeValue(self.clusterToClusterNode[c], cAnalysis.dateEnd)
self.clusterGraph.getDoubleProperty("numberOfDocuments").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbDocuments)
self.clusterGraph.getDoubleProperty("numberOfDocEdges").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbDocLinks)
self.clusterGraph.getDoubleProperty("numberOfDescriptors").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbTypes)
self.clusterGraph.getDoubleProperty("numberOfDescEdges").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbTypeLinks)
self.clusterGraph.getDoubleProperty("documentsDensity").setNodeValue(self.clusterToClusterNode[c], cAnalysis.documentsDensity)
self.clusterGraph.getDoubleProperty("descriptorsDensity").setNodeValue(self.clusterToClusterNode[c], cAnalysis.descriptorsDensity)
self.clusterGraph.getLayoutProperty("gravityCenter").setNodeValue(self.clusterToClusterNode[c], tlp.Coord(cAnalysis.gravityCenter[0],cAnalysis.gravityCenter[1],cAnalysis.gravityCenter[2]))
self.clusterGraph.getBooleanProperty("isComplex").setNodeValue(self.clusterToClusterNode[c], cAnalysis.isComplex)
#self.detailGraph.createMetaNode(self.clusterToSubGraph[c])
_graph.delSubGraph(cGraph)
#create links between clusters
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution after loop: ",currentTime
self.createClusterProximities()
currentTime = datetime.datetime.now()- starttime
starttime = datetime.datetime.now()
print "execution after building proximities: ",currentTime
self.dumpToFile("clusterDump.csv")
currentTime = datetime.datetime.now() - starttime
starttime = datetime.datetime.now()
print "execution after file dump: ",currentTime
self.clusterGraphVisu()
currentTime = datetime.datetime.now()- starttime
starttime = datetime.datetime.now()
print "execution after visualisation: ",currentTime
def compute(self, graph, secondRound):
#print "calling compute"
#self.descGraph = graph
self.brewerScale = [tlp.Color(255, 255, 229), tlp.Color(255, 247, 188), tlp.Color(254, 227, 145), tlp.Color(254, 196, 79), tlp.Color(254, 153, 41), tlp.Color(236, 112, 20), tlp.Color(204, 76, 2), tlp.Color(153, 52, 4), tlp.Color(102, 37, 6)]
viewColorDoc = self.docGraph.getColorProperty("viewColor")
viewColorDesc = self.descGraph.getColorProperty("viewColor")
viewColorBcpDoc = self.docGraph.getColorProperty("viewColorBcp")
viewColorBcpDesc = self.descGraph.getColorProperty("viewColorBcp")
viewBorderColorDoc = self.docGraph.getColorProperty("viewBorderColor")
viewBorderColorDesc = self.descGraph.getColorProperty("viewBorderColor")
viewBorderWidthDoc = self.docGraph.getDoubleProperty("viewBorderWidth")
viewBorderWidthDesc = self.descGraph.getDoubleProperty("viewBorderWidth")
viewLayoutDoc = self.docGraph.getLayoutProperty("viewLayout")
viewLayoutDesc = self.descGraph.getLayoutProperty("viewLayout")
viewLabelDoc = self.docGraph.getStringProperty("viewLabel")
viewLabelDesc = self.descGraph.getStringProperty("viewLabel")
labelDoc = self.docGraph.getStringProperty("titre_propre")
labelDesc = self.descGraph.getStringProperty("typeName")
basicNode = tlp.Color(125,164,200,50)
basicEdge = tlp.Color(229,206,160,50)
basicBorder = tlp.Color(0,0,0,255)
basicWidth = 0
basicDepth = 0
for n in self.docGraph.getNodes():
viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n))
viewBorderColorDoc.setNodeValue(n, basicBorder)
viewBorderWidthDoc.setNodeValue(n, basicWidth)
pos = viewLayoutDoc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDoc.setNodeValue(n, pos)
viewLabelDoc[n] = ""#labelDoc[n]
for e in self.docGraph.getEdges():
viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e))
#viewBorderColorDoc.setEdgeValue(e, basicBorder)
viewBorderWidthDoc.setEdgeValue(e, basicWidth)
for n in self.descGraph.getNodes():
viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n))
viewBorderColorDesc.setNodeValue(n, basicBorder)
viewBorderWidthDesc.setNodeValue(n, basicWidth)
pos = viewLayoutDesc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDesc.setNodeValue(n, pos)
viewLabelDesc[n] = ""#labelDesc[n]
for e in self.descGraph.getEdges():
viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e))
#viewBorderColorDesc.setEdgeValue(e, basicBorder)
viewBorderWidthDesc.setEdgeValue(e, basicWidth)
#if self.docGraph.getName() == "Documents":
if graph.getName() == "Descriptors":
#self.descGraph = graph
selectionType = "Documents Links"#self.dataSet["Descriptors Selection"] .getCurrentString()
typeName = self.descGraph.getStringProperty("typeName")
descSel = self.descGraph.getBooleanProperty("viewSelection")
selectedDesc = []
selectedNodes = []
typeS = False
typeN = ""
for n in self.descGraph.getNodes():
typeS = descSel.getNodeValue(n)
typeN = typeName.getNodeValue(n)
if typeS and typeN != "":
selectedDesc.append(typeN)
selectedNodes.append(n)
if not len(selectedDesc):
basicNode = tlp.Color(125,164,200,150)
basicEdge = tlp.Color(229,206,160,150)
basicBorder = tlp.Color(0,0,0,255)
basicWidth = 0
basicDepth = 0
for n in self.docGraph.getNodes():
viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n))
viewBorderColorDoc.setNodeValue(n, basicBorder)
viewBorderWidthDoc.setNodeValue(n, basicWidth)
pos = viewLayoutDoc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDoc.setNodeValue(n, pos)
viewLabelDoc[n] = labelDoc[n]
for e in self.docGraph.getEdges():
viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e))
#viewBorderColorDoc.setEdgeValue(e, basicBorder)
viewBorderWidthDoc.setEdgeValue(e, basicWidth)
for n in self.descGraph.getNodes():
viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n))
viewBorderColorDesc.setNodeValue(n, basicBorder)
viewBorderWidthDesc.setNodeValue(n, basicWidth)
pos = viewLayoutDesc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDesc.setNodeValue(n, pos)
viewLabelDesc[n] = labelDesc[n]
for e in self.descGraph.getEdges():
viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e))
#viewBorderColorDesc.setEdgeValue(e, basicBorder)
viewBorderWidthDesc.setEdgeValue(e, basicWidth)
return True
docSel = self.docGraph.getBooleanProperty("viewSelection")
docSel.setAllNodeValue(False)
docSel.setAllEdgeValue(False)
for n in selectedNodes:
descSel.setNodeValue(n,True)
#selectionType = self.dataSet["Descriptors Selection"] .getCurrentString()
#print selectionType
if selectionType == "Documents Links":
itMax = self.docGraph.numberOfEdges()
if selectionType == "Documents Nodes":
itMax = self.docGraph.numberOfNodes()
if selectionType == "Documents Links and Nodes":
itMax = self.docGraph.numberOfEdges() + self.docGraph.numberOfNodes()
itCur = 0
descP = self.docGraph.getStringProperty("descripteurs")
if selectionType == "Documents Links" or selectionType == "Documents Links and Nodes":
#self.pluginProgress.progress(itCur, itMax)
itCur = itCur + 1
for e in self.docGraph.getEdges():
dList = descP[e].split(";")
for d in selectedDesc:
#dSimilarity = self.docGraph.getDoubleProperty(d+"Similarity")
#if (docSel.getEdgeValue(e) == False and dSimilarity.getEdgeValue(e) > 0):
if (docSel.getEdgeValue(e) == False and d in dList):
nodeIn = self.docGraph.source(e)
nodeOut = self.docGraph.target(e)
docSel.setNodeValue(nodeIn, True)
docSel.setNodeValue(nodeOut, True)
docSel.setEdgeValue(e, True)
break;
if selectionType == "Documents Nodes" or selectionType == "Documents Links and Nodes":
figures = [0,1,2,3,4,5,6,7,8,9]
#self.pluginProgress.progress(itCur, itMax)
itCur = itCur + 1
for n in self.docGraph.getNodes() :
if not docSel.getNodeValue(n) :
#if not self.booleanResult.getNodeValue(n) :
descList = descP.getNodeValue(n).replace("'",";").replace("-","")
descList = descList.split(";")
for i in range(len(descList)):
word = descList[i]
if word != "" and word[0] in figures:
descList[i] = "_"+word
for d in descList:
if d in selectedDesc:
docSel.setNodeValue(n,True)
#print True
break;
#if self.docGraph.getName() == "Descriptors":
if graph.getName() == "Documents":
#docGraph = self.docGraph
#self.descGraph = self.docGraph
#self.docGraph = graph
selectionType = "From Links" #self.dataSet["Documents Selection"] .getCurrentString()
descP = self.docGraph.getStringProperty("descripteurs")
selection = self.docGraph.getBooleanProperty("viewSelection")
nDescList = []
eDescList = []
figures = [0,1,2,3,4,5,6,7,8,9]
for n in selection.getNodesEqualTo(True, self.docGraph) :
#if not docSel.getNodeValue(n) :
#if not self.booleanResult.getNodeValue(n) :
descList = descP.getNodeValue(n).replace("'",";").replace("-","").split(";")
for i in range(len(descList)):
word = descList[i]
if word != "" and word[0] in figures:
descList[i] = "_"+word
nDescList.extend(descList)
nDescList = set(nDescList)
if selectionType == "From Links" or selectionType == "Shared in Links":
for e in selection.getEdgesEqualTo(True, self.docGraph) :
dList = descP[e].split(";")
if len(nDescList) == len(eDescList):
break;
for d in nDescList:
if not d in eDescList:
#pSim = self.docGraph.getDoubleProperty(d+"Similarity")
#if pSim.getEdgeValue(e) > 0:
if d in dList:
eDescList.append(d)
typeName = self.descGraph.getStringProperty("typeName")
descSel = self.descGraph.getBooleanProperty("viewSelection")
for n in self.descGraph.getNodes():
descSel.setNodeValue(n, False)
for e in self.descGraph.getEdges():
descSel.setEdgeValue(e, False)
if selectionType == "From Nodes":
for n in self.descGraph.getNodes():
if not descSel.getNodeValue(n):
cType = typeName.getNodeValue(n)
if cType in nDescList:
descSel.setNodeValue(n, True)
if selectionType == "From Links":
for n in self.descGraph.getNodes():
if not descSel.getNodeValue(n):
cType = typeName.getNodeValue(n)
if cType in eDescList:
descSel.setNodeValue(n, True)
if selectionType == "Shared in Links":
typeToEdges = {}
for d in eDescList:
typeToEdges[d] = []
for e in selection.getEdgesEqualTo(True, self.docGraph) :
dList = descP[e].split(";")
for d in eDescList:
#dSim = self.docGraph.getDoubleProperty(d+"Similarity")
#if dSim.getEdgeValue(e)>0:
if d in dList:
typeToEdges[d].append(e)
for e in self.descGraph.getEdges():
s = self.descGraph.source(e)
t = self.descGraph.target(e)
sType = typeName.getNodeValue(s)
tType = typeName.getNodeValue(t)
if sType in eDescList and tType in eDescList:
sList = typeToEdges[sType]
tList = typeToEdges[tType]
for d in sList:
if d in tList:
descSel.setNodeValue(s, True)
descSel.setNodeValue(t, True)
descSel.setEdgeValue(e, True)
break;
if selectionType == "Shared in Nodes":
typeToNodes = {}
for d in nDescList:
typeToNodes[d] = []
for n in selection.getNodesEqualTo(True, self.docGraph) :
descList = descP.getNodeValue(n)
#print descList.decode("UTF-8","replace")
#print descList.encode("UTF-8","replace")
descList = descList.replace("'",";").replace("-","").split(";")
for i in range(len(descList)):
word = descList[i]
if word != "" and word[0] in figures:
descList[i] = "_"+word
for d in descList:
if d in nDescList:
typeToNodes[d].append(n)
#print sys.getdefaultencoding()
#print typeToNodes
for e in self.descGraph.getEdges():
s = self.descGraph.source(e)
t = self.descGraph.target(e)
sType = typeName.getNodeValue(s)
tType = typeName.getNodeValue(t)
if sType in nDescList and tType in nDescList:
sList = typeToNodes[sType]
tList = typeToNodes[tType]
for d in sList:
if d in tList:
descSel.setNodeValue(s, True)
descSel.setNodeValue(t, True)
descSel.setEdgeValue(e, True)
break;
brothers = graph.getSuperGraph().getSubGraphs()
docGraph = 0
descGraph = 0
for s in brothers:
if s.getName() == "Descriptors":
descGraph = s
if s.getName() == "Documents":
docGraph = s
viewS = descGraph.getBooleanProperty("viewSelection")
typeNP = descGraph.getStringProperty("typeName")
descList = [typeNP.getNodeValue(n) for n in descGraph.getNodes() if viewS.getNodeValue(n)]
viewSDoc = docGraph.getBooleanProperty("viewSelection")
viewSDesc = descGraph.getBooleanProperty("viewSelection")
tmpGraph = docGraph.inducedSubGraph([n for n in docGraph.getNodes() if viewSDoc.getNodeValue(n)])
tmpDescGraph = descGraph.inducedSubGraph([n for n in descGraph.getNodes() if viewSDesc.getNodeValue(n)])
c = clusterAnalysisLgt(tmpGraph, descList)
print "DESCLIST:",descList
newNodeColor = tlp.Color(255,0,0,255)
if c.nbTypeComponents == 1:
#k=int((1-c.globalCoherence)*255)
#newNodeColor[0] = k
#newNodeColor[1] = k
#newNodeColor[2] = k
#newNodeColor[3] = 255
k = int(c.globalCoherence*8.999)
newNodeColor = self.brewerScale[k]
newBorderWidth = 2
newDepth = 100
if secondRound:
newDepth = 50
newBorderColor = tlp.Color(0,150,150,255)#tlp.Color(0,0,0,255)
#if not secondRound:
# newBorderColor = tlp.Color(0,150,150,255)
for n in tmpGraph.getNodes():
viewColorDoc.setNodeValue(n, newNodeColor)
viewBorderWidthDoc.setNodeValue(n,newBorderWidth)
viewBorderColorDoc.setNodeValue(n,newBorderColor)
pos = viewLayoutDoc.getNodeValue(n)
pos[2] = newDepth
viewLayoutDoc.setNodeValue(n, pos)
viewLabelDoc[n] = labelDoc[n]
for e in tmpGraph.getEdges():
viewColorDoc.setEdgeValue(e, newNodeColor)
#viewBorderWidthDoc.setEdgeValue(e,newBorderWidth)
viewBorderColorDoc.setEdgeValue(e,newBorderColor)
if secondRound:
viewSDoc.setAllNodeValue(False)
viewSDoc.setAllEdgeValue(False)
if graph.getName() == "Descriptors":
strCol = tlp.StringCollection()
strCol.push_back("From Links")
strCol.setCurrent("From Links")
#self.dataSet["Documents Selection"] = strCol
#self.compute(docGraph, True)
if secondRound:
newBorderColor = tlp.Color(0,0,0,255)
for n in tmpDescGraph.getNodes():
viewColorDesc.setNodeValue(n, newNodeColor)
viewBorderWidthDesc.setNodeValue(n,newBorderWidth)
viewBorderColorDesc.setNodeValue(n,newBorderColor)
#viewSDesc.setNodeValue(n,False)
pos = viewLayoutDesc.getNodeValue(n)
pos[2] = newDepth
viewLayoutDesc.setNodeValue(n, pos)
viewLabelDesc[n] = labelDesc[n]
for e in tmpDescGraph.getEdges():
viewColorDesc.setEdgeValue(e, newNodeColor)
viewBorderWidthDesc.setEdgeValue(e,newBorderWidth)
#viewBorderColorDoc.setEdgeValue(e,newBorderColor)
#viewSDesc.setEdgeValue(e,False)
viewSDesc.setAllNodeValue(False)
viewSDesc.setAllEdgeValue(False)
if len(descList) == 0:
basicNode = tlp.Color(125,164,200,150)
basicEdge = tlp.Color(229,206,160,150)
basicBorder = tlp.Color(0,0,0,255)
basicWidth = 0
basicDepth = 0
for n in self.docGraph.getNodes():
viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n))
viewBorderColorDoc.setNodeValue(n, basicBorder)
viewBorderWidthDoc.setNodeValue(n, basicWidth)
pos = viewLayoutDoc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDoc.setNodeValue(n, pos)
viewLabelDoc[n] = labelDoc[n]
for e in self.docGraph.getEdges():
viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e))
#viewBorderColorDoc.setEdgeValue(e, basicBorder)
viewBorderWidthDoc.setEdgeValue(e, basicWidth)
for n in self.descGraph.getNodes():
viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n))
viewBorderColorDesc.setNodeValue(n, basicBorder)
viewBorderWidthDesc.setNodeValue(n, basicWidth)
pos = viewLayoutDesc.getNodeValue(n)
pos[2] = basicDepth
viewLayoutDesc.setNodeValue(n, pos)
viewLabelDesc[n] = labelDesc[n]
for e in self.descGraph.getEdges():
viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e))
#viewBorderColorDesc.setEdgeValue(e, basicBorder)
viewBorderWidthDesc.setEdgeValue(e, basicWidth)
docGraph.delSubGraph(tmpGraph)
descGraph.delSubGraph(tmpDescGraph)
print "\nREPORT:\n"
print descList
print "coherence ", c.globalCoherence
print "cosine ",c.globalCosine
print "gamma_i ",c.typeToIntrications
print "raw Matrix ",c.rawMatrix
print "c Matrix ",c.cMatrix
print "nb components ", c.nbTypeComponents
print c.typeIsConnected
print c.typeList
