def __init__(self, _graph, _descriptorListFile, _clusterProperty): starttime = datetime.datetime.now() self.layoutAlgorithm = "GEM (Frick)" if _descriptorListFile != "": self.fullTypeList = self.readTypeList(_descriptorListFile) else: self.fullTypeList = set() descP = _graph.getStringProperty("descripteurs") for n in _graph.getNodes(): self.fullTypeList.update(descP.getNodeValue(n).split(';')) self.fullTypeList = list(self.fullTypeList) self.analyseGraph = tlp.newSubGraph(_graph.getSuperGraph(), _graph.getName()+" Analysis") #self.docGraph = tlp.newSubGraph(self.analyseGraph, "documents") self.clusterGraph = tlp.newSubGraph(self.analyseGraph, "clusters") self.detailGraph = tlp.newSubGraph(self.analyseGraph, "details") currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution1: ",currentTime #tlp.copyToGraph(self.docGraph, _graph) currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution1.1: ",currentTime clusterProperty = _graph.getDoubleProperty(_clusterProperty) minP = clusterProperty.getNodeMin(_graph) maxP = clusterProperty.getNodeMax(_graph) +1 currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution1.2: ",currentTime self.clusterToNodes = {} for c in range(int(minP), int(maxP)): self.clusterToNodes[c] = [] currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution1.3: ",currentTime for n in _graph.getNodes(): cluster = clusterProperty.getNodeValue(n) self.clusterToNodes[int(cluster)].append(n) #print clusterToNodes currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution1.4: ",currentTime currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution2: ",currentTime self.clusterToClusterNode = {} #nodes in the cluster subgraph corresponding to the cluster self.clusterToSubGraph = {} #subgraph in the detail subgraph corresponding to the cluster self.clusterToTypeList = {} clusterCount = 0 for c in self.clusterToNodes.keys(): #if c >1: # break print "handling cluster ",clusterCount clusterCount = clusterCount+1 currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution: ",currentTime clustercount= clusterCount+1 self.clusterToClusterNode[c] = self.clusterGraph.addNode() cGraph =_graph.inducedSubGraph(self.clusterToNodes[c]) # builds the subgraph hierarchy self.clusterToSubGraph[c] = [tlp.newSubGraph(self.detailGraph, "%d"%c)] self.clusterToSubGraph[c].append(tlp.newSubGraph(self.clusterToSubGraph[c][0], "Documents")) self.clusterToSubGraph[c].append(tlp.newSubGraph(self.clusterToSubGraph[c][0], "Descriptors")) self.copyToGraph(self.clusterToSubGraph[c][1], cGraph) #analyse the graph cAnalysis = clusterAnalysisLgt(cGraph, self.fullTypeList) cAnalysis.typeGraph.computeLayoutProperty(self.layoutAlgorithm,cAnalysis.typeGraph.getLayoutProperty("viewLayout")) tlp.copyToGraph(self.clusterToSubGraph[c][2], cAnalysis.typeGraph) self.clusterToTypeList[c] = cAnalysis.typeList title = " ".join(cAnalysis.typeList) self.clusterToSubGraph[c][0].setName(title) self.clusterToSubGraph[c][0].computeLayoutProperty("Connected Component Packing", self.clusterToSubGraph[c][0].getLayoutProperty("viewLayout")) self.clusterGraph.getStringProperty("viewLabel").setNodeValue(self.clusterToClusterNode[c], title) self.clusterGraph.getStringProperty("clusterName").setNodeValue(self.clusterToClusterNode[c], title) self.clusterGraph.getStringProperty("clusterTypeList").setNodeValue(self.clusterToClusterNode[c], title) self.clusterGraph.getDoubleProperty("clusterCoherence").setNodeValue(self.clusterToClusterNode[c], cAnalysis.globalCoherence) self.clusterGraph.getDoubleProperty("clusterGraphID").setNodeValue(self.clusterToClusterNode[c], self.clusterToSubGraph[c][0].getId()) self.clusterGraph.getDoubleProperty("clusterCosine").setNodeValue(self.clusterToClusterNode[c], cAnalysis.globalCosine) self.clusterGraph.getDoubleProperty("dateBegin").setNodeValue(self.clusterToClusterNode[c], cAnalysis.dateBegin) self.clusterGraph.getDoubleProperty("dateEnd").setNodeValue(self.clusterToClusterNode[c], cAnalysis.dateEnd) self.clusterGraph.getDoubleProperty("numberOfDocuments").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbDocuments) self.clusterGraph.getDoubleProperty("numberOfDocEdges").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbDocLinks) self.clusterGraph.getDoubleProperty("numberOfDescriptors").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbTypes) self.clusterGraph.getDoubleProperty("numberOfDescEdges").setNodeValue(self.clusterToClusterNode[c], cAnalysis.nbTypeLinks) self.clusterGraph.getDoubleProperty("documentsDensity").setNodeValue(self.clusterToClusterNode[c], cAnalysis.documentsDensity) self.clusterGraph.getDoubleProperty("descriptorsDensity").setNodeValue(self.clusterToClusterNode[c], cAnalysis.descriptorsDensity) self.clusterGraph.getLayoutProperty("gravityCenter").setNodeValue(self.clusterToClusterNode[c], tlp.Coord(cAnalysis.gravityCenter[0],cAnalysis.gravityCenter[1],cAnalysis.gravityCenter[2])) self.clusterGraph.getBooleanProperty("isComplex").setNodeValue(self.clusterToClusterNode[c], cAnalysis.isComplex) #self.detailGraph.createMetaNode(self.clusterToSubGraph[c]) _graph.delSubGraph(cGraph) #create links between clusters currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution after loop: ",currentTime self.createClusterProximities() currentTime = datetime.datetime.now()- starttime starttime = datetime.datetime.now() print "execution after building proximities: ",currentTime self.dumpToFile("clusterDump.csv") currentTime = datetime.datetime.now() - starttime starttime = datetime.datetime.now() print "execution after file dump: ",currentTime self.clusterGraphVisu() currentTime = datetime.datetime.now()- starttime starttime = datetime.datetime.now() print "execution after visualisation: ",currentTime
def compute(self, graph, secondRound): #print "calling compute" #self.descGraph = graph self.brewerScale = [tlp.Color(255, 255, 229), tlp.Color(255, 247, 188), tlp.Color(254, 227, 145), tlp.Color(254, 196, 79), tlp.Color(254, 153, 41), tlp.Color(236, 112, 20), tlp.Color(204, 76, 2), tlp.Color(153, 52, 4), tlp.Color(102, 37, 6)] viewColorDoc = self.docGraph.getColorProperty("viewColor") viewColorDesc = self.descGraph.getColorProperty("viewColor") viewColorBcpDoc = self.docGraph.getColorProperty("viewColorBcp") viewColorBcpDesc = self.descGraph.getColorProperty("viewColorBcp") viewBorderColorDoc = self.docGraph.getColorProperty("viewBorderColor") viewBorderColorDesc = self.descGraph.getColorProperty("viewBorderColor") viewBorderWidthDoc = self.docGraph.getDoubleProperty("viewBorderWidth") viewBorderWidthDesc = self.descGraph.getDoubleProperty("viewBorderWidth") viewLayoutDoc = self.docGraph.getLayoutProperty("viewLayout") viewLayoutDesc = self.descGraph.getLayoutProperty("viewLayout") viewLabelDoc = self.docGraph.getStringProperty("viewLabel") viewLabelDesc = self.descGraph.getStringProperty("viewLabel") labelDoc = self.docGraph.getStringProperty("titre_propre") labelDesc = self.descGraph.getStringProperty("typeName") basicNode = tlp.Color(125,164,200,50) basicEdge = tlp.Color(229,206,160,50) basicBorder = tlp.Color(0,0,0,255) basicWidth = 0 basicDepth = 0 for n in self.docGraph.getNodes(): viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n)) viewBorderColorDoc.setNodeValue(n, basicBorder) viewBorderWidthDoc.setNodeValue(n, basicWidth) pos = viewLayoutDoc.getNodeValue(n) pos[2] = basicDepth viewLayoutDoc.setNodeValue(n, pos) viewLabelDoc[n] = ""#labelDoc[n] for e in self.docGraph.getEdges(): viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e)) #viewBorderColorDoc.setEdgeValue(e, basicBorder) viewBorderWidthDoc.setEdgeValue(e, basicWidth) for n in self.descGraph.getNodes(): viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n)) viewBorderColorDesc.setNodeValue(n, basicBorder) viewBorderWidthDesc.setNodeValue(n, basicWidth) pos = viewLayoutDesc.getNodeValue(n) pos[2] = basicDepth viewLayoutDesc.setNodeValue(n, pos) viewLabelDesc[n] = ""#labelDesc[n] for e in self.descGraph.getEdges(): viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e)) #viewBorderColorDesc.setEdgeValue(e, basicBorder) viewBorderWidthDesc.setEdgeValue(e, basicWidth) #if self.docGraph.getName() == "Documents": if graph.getName() == "Descriptors": #self.descGraph = graph selectionType = "Documents Links"#self.dataSet["Descriptors Selection"] .getCurrentString() typeName = self.descGraph.getStringProperty("typeName") descSel = self.descGraph.getBooleanProperty("viewSelection") selectedDesc = [] selectedNodes = [] typeS = False typeN = "" for n in self.descGraph.getNodes(): typeS = descSel.getNodeValue(n) typeN = typeName.getNodeValue(n) if typeS and typeN != "": selectedDesc.append(typeN) selectedNodes.append(n) if not len(selectedDesc): basicNode = tlp.Color(125,164,200,150) basicEdge = tlp.Color(229,206,160,150) basicBorder = tlp.Color(0,0,0,255) basicWidth = 0 basicDepth = 0 for n in self.docGraph.getNodes(): viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n)) viewBorderColorDoc.setNodeValue(n, basicBorder) viewBorderWidthDoc.setNodeValue(n, basicWidth) pos = viewLayoutDoc.getNodeValue(n) pos[2] = basicDepth viewLayoutDoc.setNodeValue(n, pos) viewLabelDoc[n] = labelDoc[n] for e in self.docGraph.getEdges(): viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e)) #viewBorderColorDoc.setEdgeValue(e, basicBorder) viewBorderWidthDoc.setEdgeValue(e, basicWidth) for n in self.descGraph.getNodes(): viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n)) viewBorderColorDesc.setNodeValue(n, basicBorder) viewBorderWidthDesc.setNodeValue(n, basicWidth) pos = viewLayoutDesc.getNodeValue(n) pos[2] = basicDepth viewLayoutDesc.setNodeValue(n, pos) viewLabelDesc[n] = labelDesc[n] for e in self.descGraph.getEdges(): viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e)) #viewBorderColorDesc.setEdgeValue(e, basicBorder) viewBorderWidthDesc.setEdgeValue(e, basicWidth) return True docSel = self.docGraph.getBooleanProperty("viewSelection") docSel.setAllNodeValue(False) docSel.setAllEdgeValue(False) for n in selectedNodes: descSel.setNodeValue(n,True) #selectionType = self.dataSet["Descriptors Selection"] .getCurrentString() #print selectionType if selectionType == "Documents Links": itMax = self.docGraph.numberOfEdges() if selectionType == "Documents Nodes": itMax = self.docGraph.numberOfNodes() if selectionType == "Documents Links and Nodes": itMax = self.docGraph.numberOfEdges() + self.docGraph.numberOfNodes() itCur = 0 descP = self.docGraph.getStringProperty("descripteurs") if selectionType == "Documents Links" or selectionType == "Documents Links and Nodes": #self.pluginProgress.progress(itCur, itMax) itCur = itCur + 1 for e in self.docGraph.getEdges(): dList = descP[e].split(";") for d in selectedDesc: #dSimilarity = self.docGraph.getDoubleProperty(d+"Similarity") #if (docSel.getEdgeValue(e) == False and dSimilarity.getEdgeValue(e) > 0): if (docSel.getEdgeValue(e) == False and d in dList): nodeIn = self.docGraph.source(e) nodeOut = self.docGraph.target(e) docSel.setNodeValue(nodeIn, True) docSel.setNodeValue(nodeOut, True) docSel.setEdgeValue(e, True) break; if selectionType == "Documents Nodes" or selectionType == "Documents Links and Nodes": figures = [0,1,2,3,4,5,6,7,8,9] #self.pluginProgress.progress(itCur, itMax) itCur = itCur + 1 for n in self.docGraph.getNodes() : if not docSel.getNodeValue(n) : #if not self.booleanResult.getNodeValue(n) : descList = descP.getNodeValue(n).replace("'",";").replace("-","") descList = descList.split(";") for i in range(len(descList)): word = descList[i] if word != "" and word[0] in figures: descList[i] = "_"+word for d in descList: if d in selectedDesc: docSel.setNodeValue(n,True) #print True break; #if self.docGraph.getName() == "Descriptors": if graph.getName() == "Documents": #docGraph = self.docGraph #self.descGraph = self.docGraph #self.docGraph = graph selectionType = "From Links" #self.dataSet["Documents Selection"] .getCurrentString() descP = self.docGraph.getStringProperty("descripteurs") selection = self.docGraph.getBooleanProperty("viewSelection") nDescList = [] eDescList = [] figures = [0,1,2,3,4,5,6,7,8,9] for n in selection.getNodesEqualTo(True, self.docGraph) : #if not docSel.getNodeValue(n) : #if not self.booleanResult.getNodeValue(n) : descList = descP.getNodeValue(n).replace("'",";").replace("-","").split(";") for i in range(len(descList)): word = descList[i] if word != "" and word[0] in figures: descList[i] = "_"+word nDescList.extend(descList) nDescList = set(nDescList) if selectionType == "From Links" or selectionType == "Shared in Links": for e in selection.getEdgesEqualTo(True, self.docGraph) : dList = descP[e].split(";") if len(nDescList) == len(eDescList): break; for d in nDescList: if not d in eDescList: #pSim = self.docGraph.getDoubleProperty(d+"Similarity") #if pSim.getEdgeValue(e) > 0: if d in dList: eDescList.append(d) typeName = self.descGraph.getStringProperty("typeName") descSel = self.descGraph.getBooleanProperty("viewSelection") for n in self.descGraph.getNodes(): descSel.setNodeValue(n, False) for e in self.descGraph.getEdges(): descSel.setEdgeValue(e, False) if selectionType == "From Nodes": for n in self.descGraph.getNodes(): if not descSel.getNodeValue(n): cType = typeName.getNodeValue(n) if cType in nDescList: descSel.setNodeValue(n, True) if selectionType == "From Links": for n in self.descGraph.getNodes(): if not descSel.getNodeValue(n): cType = typeName.getNodeValue(n) if cType in eDescList: descSel.setNodeValue(n, True) if selectionType == "Shared in Links": typeToEdges = {} for d in eDescList: typeToEdges[d] = [] for e in selection.getEdgesEqualTo(True, self.docGraph) : dList = descP[e].split(";") for d in eDescList: #dSim = self.docGraph.getDoubleProperty(d+"Similarity") #if dSim.getEdgeValue(e)>0: if d in dList: typeToEdges[d].append(e) for e in self.descGraph.getEdges(): s = self.descGraph.source(e) t = self.descGraph.target(e) sType = typeName.getNodeValue(s) tType = typeName.getNodeValue(t) if sType in eDescList and tType in eDescList: sList = typeToEdges[sType] tList = typeToEdges[tType] for d in sList: if d in tList: descSel.setNodeValue(s, True) descSel.setNodeValue(t, True) descSel.setEdgeValue(e, True) break; if selectionType == "Shared in Nodes": typeToNodes = {} for d in nDescList: typeToNodes[d] = [] for n in selection.getNodesEqualTo(True, self.docGraph) : descList = descP.getNodeValue(n) #print descList.decode("UTF-8","replace") #print descList.encode("UTF-8","replace") descList = descList.replace("'",";").replace("-","").split(";") for i in range(len(descList)): word = descList[i] if word != "" and word[0] in figures: descList[i] = "_"+word for d in descList: if d in nDescList: typeToNodes[d].append(n) #print sys.getdefaultencoding() #print typeToNodes for e in self.descGraph.getEdges(): s = self.descGraph.source(e) t = self.descGraph.target(e) sType = typeName.getNodeValue(s) tType = typeName.getNodeValue(t) if sType in nDescList and tType in nDescList: sList = typeToNodes[sType] tList = typeToNodes[tType] for d in sList: if d in tList: descSel.setNodeValue(s, True) descSel.setNodeValue(t, True) descSel.setEdgeValue(e, True) break; brothers = graph.getSuperGraph().getSubGraphs() docGraph = 0 descGraph = 0 for s in brothers: if s.getName() == "Descriptors": descGraph = s if s.getName() == "Documents": docGraph = s viewS = descGraph.getBooleanProperty("viewSelection") typeNP = descGraph.getStringProperty("typeName") descList = [typeNP.getNodeValue(n) for n in descGraph.getNodes() if viewS.getNodeValue(n)] viewSDoc = docGraph.getBooleanProperty("viewSelection") viewSDesc = descGraph.getBooleanProperty("viewSelection") tmpGraph = docGraph.inducedSubGraph([n for n in docGraph.getNodes() if viewSDoc.getNodeValue(n)]) tmpDescGraph = descGraph.inducedSubGraph([n for n in descGraph.getNodes() if viewSDesc.getNodeValue(n)]) c = clusterAnalysisLgt(tmpGraph, descList) print "DESCLIST:",descList newNodeColor = tlp.Color(255,0,0,255) if c.nbTypeComponents == 1: #k=int((1-c.globalCoherence)*255) #newNodeColor[0] = k #newNodeColor[1] = k #newNodeColor[2] = k #newNodeColor[3] = 255 k = int(c.globalCoherence*8.999) newNodeColor = self.brewerScale[k] newBorderWidth = 2 newDepth = 100 if secondRound: newDepth = 50 newBorderColor = tlp.Color(0,150,150,255)#tlp.Color(0,0,0,255) #if not secondRound: # newBorderColor = tlp.Color(0,150,150,255) for n in tmpGraph.getNodes(): viewColorDoc.setNodeValue(n, newNodeColor) viewBorderWidthDoc.setNodeValue(n,newBorderWidth) viewBorderColorDoc.setNodeValue(n,newBorderColor) pos = viewLayoutDoc.getNodeValue(n) pos[2] = newDepth viewLayoutDoc.setNodeValue(n, pos) viewLabelDoc[n] = labelDoc[n] for e in tmpGraph.getEdges(): viewColorDoc.setEdgeValue(e, newNodeColor) #viewBorderWidthDoc.setEdgeValue(e,newBorderWidth) viewBorderColorDoc.setEdgeValue(e,newBorderColor) if secondRound: viewSDoc.setAllNodeValue(False) viewSDoc.setAllEdgeValue(False) if graph.getName() == "Descriptors": strCol = tlp.StringCollection() strCol.push_back("From Links") strCol.setCurrent("From Links") #self.dataSet["Documents Selection"] = strCol #self.compute(docGraph, True) if secondRound: newBorderColor = tlp.Color(0,0,0,255) for n in tmpDescGraph.getNodes(): viewColorDesc.setNodeValue(n, newNodeColor) viewBorderWidthDesc.setNodeValue(n,newBorderWidth) viewBorderColorDesc.setNodeValue(n,newBorderColor) #viewSDesc.setNodeValue(n,False) pos = viewLayoutDesc.getNodeValue(n) pos[2] = newDepth viewLayoutDesc.setNodeValue(n, pos) viewLabelDesc[n] = labelDesc[n] for e in tmpDescGraph.getEdges(): viewColorDesc.setEdgeValue(e, newNodeColor) viewBorderWidthDesc.setEdgeValue(e,newBorderWidth) #viewBorderColorDoc.setEdgeValue(e,newBorderColor) #viewSDesc.setEdgeValue(e,False) viewSDesc.setAllNodeValue(False) viewSDesc.setAllEdgeValue(False) if len(descList) == 0: basicNode = tlp.Color(125,164,200,150) basicEdge = tlp.Color(229,206,160,150) basicBorder = tlp.Color(0,0,0,255) basicWidth = 0 basicDepth = 0 for n in self.docGraph.getNodes(): viewColorDoc.setNodeValue(n, viewColorBcpDoc.getNodeValue(n)) viewBorderColorDoc.setNodeValue(n, basicBorder) viewBorderWidthDoc.setNodeValue(n, basicWidth) pos = viewLayoutDoc.getNodeValue(n) pos[2] = basicDepth viewLayoutDoc.setNodeValue(n, pos) viewLabelDoc[n] = labelDoc[n] for e in self.docGraph.getEdges(): viewColorDoc.setEdgeValue(e, viewColorBcpDoc.getEdgeValue(e)) #viewBorderColorDoc.setEdgeValue(e, basicBorder) viewBorderWidthDoc.setEdgeValue(e, basicWidth) for n in self.descGraph.getNodes(): viewColorDesc.setNodeValue(n, viewColorBcpDesc.getNodeValue(n)) viewBorderColorDesc.setNodeValue(n, basicBorder) viewBorderWidthDesc.setNodeValue(n, basicWidth) pos = viewLayoutDesc.getNodeValue(n) pos[2] = basicDepth viewLayoutDesc.setNodeValue(n, pos) viewLabelDesc[n] = labelDesc[n] for e in self.descGraph.getEdges(): viewColorDesc.setEdgeValue(e, viewColorBcpDesc.getEdgeValue(e)) #viewBorderColorDesc.setEdgeValue(e, basicBorder) viewBorderWidthDesc.setEdgeValue(e, basicWidth) docGraph.delSubGraph(tmpGraph) descGraph.delSubGraph(tmpDescGraph) print "\nREPORT:\n" print descList print "coherence ", c.globalCoherence print "cosine ",c.globalCosine print "gamma_i ",c.typeToIntrications print "raw Matrix ",c.rawMatrix print "c Matrix ",c.cMatrix print "nb components ", c.nbTypeComponents print c.typeIsConnected print c.typeList