def importConceptsIntoArchi():
logger.info(u"Using : %s" % fileArchimateTest)
conceptFile = fileConceptsBatches
logger.info(u"Loading :" + conceptFile)
concepts = Concepts.loadConcepts(conceptFile)
al = ArchiLib()
# Create Subfolder
folder = u"Implementation & Migration"
subfolder = u"Dependancy Analysis - %s" % time.strftime(u"%Y%d%m_%H%M%S")
attrib = dict()
attrib[u"id"] = al.getID()
attrib[u"name"] = subfolder
al.insertNode(u"folder", folder, attrib)
logger.info(u"--- Insert Nodes ---")
insertConceptNode(al, concepts, subfolder)
logger.info(u"--- Insert Relations ---")
insertConceptRelation(al, concepts)
al.outputXMLtoFile(filename=u"import_concepts.archimate")
def importConceptsIntoNeo4J(fileArchimate, gdb, ClearNeo4J=True):
icnj = ConceptsImportNeo4J(fileArchimate, gdb, ClearNeo4J=ClearNeo4J)
importConcepts = Concepts.loadConcepts(fileConceptsExport)
icnj.importNeo4J(importConcepts)
def __init__(self, concepts=None):
if concepts is None:
logger.info(u"Loading : %s" % self.conceptFile)
self.concepts = Concepts.loadConcepts(self.conceptFile)
else:
logger.info(u"Using : %s" % concepts.name)
self.concepts = concepts
self.chunkConcepts = Concepts(u"Chunk", u"Chunks")
def conceptsImportNeo4J(fileArchimate, gdb):
logger.info(u"Using : %s" % fileConceptsExport)
importConcepts = Concepts.loadConcepts(fileConceptsExport)
in4j = ConceptsImportNeo4J(fileArchimateTest, gdbTest, ClearNeo4J=True)
in4j.importNeo4J(importConcepts)
def test_importConceptsIntoNeo4J(fileArchimate, gdb):
assert (os.path.isfile(fileArchimate) is True)
icnj = ConceptsImportNeo4J(fileArchimate, gdb, ClearNeo4J=True)
assert (os.path.isfile(fileConceptsExport) is True)
importConcepts = Concepts.loadConcepts(fileConceptsExport)
icnj.importNeo4J(importConcepts)
def conceptsImportNeo4J(fileArchimate, gdb):
logger.info(u"Using : %s" % fileConceptsExport)
importConcepts = Concepts.loadConcepts(fileConceptsExport)
in4j = ConceptsImportNeo4J(fileArchimateTest, gdbTest, ClearNeo4J=True)
in4j.importNeo4J(importConcepts)
def createTagCloud(conceptFile, topic):
start_time = ArchiLib.startTimer()
concepts = Concepts.loadConcepts(conceptFile)
tc = TopicCloud(concepts, font_path=u"/Users/morrj140/Fonts/DroidSans.ttf", imageFile=u"Topics.png")
tc.createTagCloud(topic)
ArchiLib.stopTimer(start_time)
def __init__(self, conceptFile=None):
if conceptFile == None:
conceptFile = u"documents.p"
logger.info(u"Load Concepts from %s " % (conceptFile))
self.concepts = Concepts.loadConcepts(conceptFile)
logger.info(u"Loaded Concepts")
self.conceptsNGram = Concepts(u"n-gram", u"NGRAM")
self.conceptsNGramScore = Concepts(u"NGram_Score", u"Score")
self.conceptsNGramSubject = Concepts(u"Subject", u"Subjects")
def createArchiFromConcepts(fileArchimate, fileConceptsImport, fileArchimateImport):
logger.info(u"Using : %s" % fileArchimate)
logger.info(u"Loading :" + fileConceptsImport)
ic = ConceptsImportArchi(fileArchimate, fileConceptsImport)
concepts = Concepts.loadConcepts(fileConceptsImport)
# Create Subfolder
folder = u"Implementation & Migration"
subfolder = u"Dependancy Analysis - %s" % time.strftime(u"%Y%d%m_%H%M%S")
ic.importConcepts(concepts, folder, subfolder)
ic.exportXML(fileArchimateImport)
def graphConcepts(graph, conceptFile):
concepts = Concepts.loadConcepts(conceptFile)
# concepts.logConcepts()
graph.addGraphNodes(concepts)
graph.addGraphEdges(concepts)
if isinstance(graph, NetworkXGraph):
graph.saveJSON(concepts)
if isinstance(graph, GraphVizGraph):
graph.exportGraph()
if isinstance(graph, PatternGraph):
graph.exportGraph()
def graphConcepts(graph, conceptFile):
concepts = Concepts.loadConcepts(conceptFile)
# concepts.logConcepts()
graph.addGraphNodes(concepts)
graph.addGraphEdges(concepts)
if isinstance(graph, NetworkXGraph):
graph.saveJSON(concepts)
if isinstance(graph, GraphVizGraph):
graph.exportGraph()
if isinstance(graph, PatternGraph):
graph.exportGraph()
def test_GraphConcepts(cleandir):
assert (os.path.isfile(fileConceptsNGramsSubject) is True)
c = Concepts(u"GraphConcepts", u"GRAPH")
concepts = Concepts.loadConcepts(fileConceptsNGramsSubject)
# c.logConcepts()
# graph = PatternGraph()
graph = GraphVizGraph()
cg = ConceptsGraph(graph=graph, fileImage=fileImageExport)
cg.conceptsGraph(concepts)
assert (os.path.isfile(fileImageExport) is True)
def analyzeNetworkX(self, concepts, fileConceptsExport=None):
if concepts is None and fileConceptsExport is not None:
concepts = Concepts.loadConcepts(fileConceptsExport)
logger.info(u" Concepts : %s[%d][%s]" % (concepts.name, len(concepts.getConcepts()), concepts.typeName))
self.graph = NetworkXGraph()
logger.info(u"Adding NetworkX nodes to the graph ...")
self.addGraphNodes(concepts)
logger.info(u"Adding NetworkX edges to the graph ...")
self.addGraphEdges(concepts)
gl = nx.pagerank(self.graph.G)
self.analyzeGraph(gl, u"PageRank")
def test_ConceptsImportArchi():
start_time = ArchiLib.startTimer()
logger.info(u"Using : %s" % fileArchimateTest)
logger.info(u"Loading :" + fileConceptsExport)
ic = ConceptsImportArchi(fileArchimateTest, fileConceptsExport)
concepts = Concepts.loadConcepts(fileConceptsExport)
# Create Subfolder
folder = u"Implementation & Migration"
subfolder = u"Dependancy Analysis - %s" % time.strftime(u"%Y%d%m_%H%M%S")
ic.importConcepts(concepts, folder, subfolder)
ic.exportXML()
ArchiLib.stopTimer(start_time)
def test_CreateArchiFromConcepts(cleandir, fileArchimate):
assert (os.path.isfile(fileArchimate) is True)
assert (os.path.isfile(fileConceptsImport) is True)
logger.info(u"Using : %s" % fileArchimate)
logger.info(u"Loading : %s" % fileConceptsImport)
ic = ConceptsImportArchi(fileArchimate, fileConceptsImport)
concepts = Concepts.loadConcepts(fileConceptsImport)
# Create Subfolder
folder = u"Implementation & Migration"
subfolder = u"Dependancy Analysis - %s" % time.strftime(u"%Y%d%m_%H%M%S")
ic.importConcepts(concepts, folder, subfolder)
ic.exportXML(fileArchimateImport)
assert (os.path.isfile(fileArchimateImport) is True)
def test_ConceptsImportArchi():
start_time = ArchiLib.startTimer()
logger.info(u"Using : %s" % fileArchimateTest)
logger.info(u"Loading :" + fileConceptsExport)
ic = ConceptsImportArchi(fileArchimateTest, fileConceptsExport)
concepts = Concepts.loadConcepts(fileConceptsExport)
# Create Subfolder
folder = u"Implementation & Migration"
subfolder = u"Dependancy Analysis - %s" % time.strftime(u"%Y%d%m_%H%M%S")
ic.importConcepts(concepts, folder, subfolder)
ic.exportXML()
ArchiLib.stopTimer(start_time)
def createTopics(self, conceptsFile, concepts=None):
if concepts is None:
logger.info(u"Load Concepts from " + conceptsFile)
self.concepts = Concepts.loadConcepts(conceptsFile)
logger.info(u"Loaded Concepts")
else:
self.concepts = concepts
self.tm = TopicsModel()
logger.info(u"Load Documents from Concepts")
self.documentsList, self.wordcount = self.tm.loadConceptsWords(self.concepts)
self.mapDocuments = { self.documentsList.index(x) : x for x in self.documentsList}
self.df = self.concepts.getConcepts().keys()
logger.info(u"Read " + str(len(self.documentsList)) + u" Documents, with " + str(self.wordcount) + u" words.")
if self.wordcount == 0:
logger.error(u"No topics to use!")
return None
logger.info(u"Compute Topics")
self.topics = self.tm.computeTopics(self.documentsList, nt=num_topics, nw=num_words)
logger.info(u"Log Topics")
self.tm.logTopics(self.topics)
self.listTopics = [unicode(x[0]).strip().replace(u"\"", u"") for x in self.topics]
logger.info(u"Saving Topics")
self.topicConcepts = self.tm.saveTopics(self.topics)
logger.info(u"Complete createTopics")
return self.concepts
def createArchimate(self, fileArchiModel, fileArchiP):
archi = Concepts.loadConcepts(fileArchiP)
rootName = etree.QName(ARCHIMATE_NS, u'model')
root = etree.Element(rootName,
version=u"2.6.0",
name=fileArchiP,
id=u"02cec69f",
nsmap=NS_MAP)
xmlSheet = etree.ElementTree(root)
self.createArchimateElements(xmlSheet, archi, root)
output = StringIO.StringIO()
xmlSheet.write(output, pretty_print=True)
logger.debug(u"%s" % (output.getvalue()))
f = open(fileArchiModel, u'w')
f.write(output.getvalue())
f.close()
output.close()
def getChunkTopics():
# conceptFile = u"documents.p"
conceptFile = u"chunks.p"
# conceptFile = u"topicsDict.p"
# conceptFile = u"TopicChunks.p"
# conceptFile = u"ngramsubject.p"
conceptPathFile = os.getcwd() + os.sep + u"run" + os.sep + conceptFile
logger.info(u"Loading :" + os.getcwd() + os.sep + u"run" + os.sep + conceptFile)
concepts = Concepts.loadConcepts(conceptPathFile)
listTopics = list()
cf = open(conceptFile[:-2] + u".txt", u"wb")
for conceptDoc in concepts.getConcepts().values():
logger.debug(u"len %d" % len(conceptDoc.getConcepts()))
if len(conceptDoc.getConcepts()) > 1:
logger.debug(u"%d:%s" % (len(conceptDoc.getConcepts()), conceptDoc.name))
listTopics.append((len(conceptDoc.getConcepts()), conceptDoc))
for concept in conceptDoc.getConcepts().values():
logger.debug(u"--->" + concept.name)
logger.info(u"---- Sorted Topics ----")
cf.write(u"---- Sorted Topics ----" + os.linesep)
lt = sorted(listTopics, key=lambda c: c[0], reverse=True)
for x in lt:
logger.info(u"%s" % x[1].name)
cf.write(x[1].name + os.linesep)
for concept in x[1].getConcepts().values():
logger.info(u"--->%s" % concept.name)
cf.write(u"--->" + concept.name + os.linesep)
cf.close()
def graphConcepts(conceptFile):
start_time = ArchiLib.startTimer()
concepts = Concepts.loadConcepts(conceptFile)
# concepts.logConcepts()
# graph = PatternGraph()
graph = GraphVizGraph()
# graph = NetworkXGraph(conceptFile[:-2]+u".png")
graph.addGraphNodes(concepts)
graph.addGraphEdges(concepts)
if isinstance(graph, NetworkXGraph):
graph.saveJSON(concepts)
if isinstance(graph, GraphVizGraph):
graph.exportGraph()
if isinstance(graph, PatternGraph):
graph.exportGraph()
ArchiLib.stopTimer(start_time)
def sortConcepts(concepts, n=0):
pc = concepts.getConcepts()
spaces = u" " * n
for p in pc.values():
if p.typeName in (u"Concept", u"Similar"):
logger.info(u"%s%s[%s] -> Count=%s" % (spaces, p.name, p.typeName, p.count))
sortConcepts(p, n+1)
if __name__ == u"__main__":
if True:
fileArchimate = u"/Users/morrj140/Documents/SolutionEngineering/Archimate Models/DVC v3.14.archimate"
searchTypes = list()
searchTypes.append(u"archimate:ApplicationComponent")
searchTypes.append(u"archimate:ApplicationFunction")
searchTypes.append(u"archimate:ApplicationService")
searchTypes.append(u"archimate:DataObject")
searchTypes.append(u"archimate:BusinessEvent")
searchTypes.append(u"archimate:BusinessProcess")
searchTypes.append(u"archimate:BusinessObject")
gapSimilarity(fileArchimate, searchTypes)
else:
conceptsSimilarityFile = u"GapsSimilarity.p"
concepts = Concepts.loadConcepts(conceptsSimilarityFile)
sortConcepts(concepts)
# conceptFile = u"batches.p"
conceptFile = u"export.p"
# conceptFile = u"reqs.p"
# conceptFile = u"Estimation.p"
#conceptFile = u"archi.p"
fileConceptsExport = u"export.csv"
if True:
filePath = conceptFile
else:
directory = os.getcwd() + os.sep + u"run" + os.sep
filePath = directory + conceptFile
logger.info(u"Loading :" + filePath)
concepts = Concepts.loadConcepts(filePath)
concepts.logConcepts()
# concepts.printConcepts(list)
# Concepts.outputConceptsToCSV(concepts, fileConceptsExport)
if False:
logger.info(u"Distribution Analysis")
tc = Concepts.loadConcepts(filePath)
distribution(concepts, tc)
distribution(concepts)
conceptFile = u"archi.p"
topic = u"name"
elif False:
conceptFile = u"ngramsubject.p"
topic = u"NGRAM"
elif False:
conceptFile = u"req.p"
topic = u"Word"
elif False:
conceptFile = u"chunks.p"
# topic = u"Lemma"
# topic = u"SBJ"
topic = u"OBJ"
# topic = u"VP"
# topic = u"NN"
# topic = u"NNP"
elif False:
conceptFile = u"ngrams.p"
topic = u"NGRAM"
c = Concepts(u"GraphConcepts", u"GRAPH")
logger.info(u"Loading Topics from : " + conceptFile)
concepts = Concepts.loadConcepts(conceptFile)
tc = TopicCloud(concepts, os.getcwd()+os.sep)
tc.createTagCloud(topic)
