def createNetFromSentences(self, sentences):
        "This function creates the network starting from a set of sentences"

        text_filter = TextFilter()
        for sentence in sentences:
            filtered_sentence = text_filter.filter_all(sentence)

            tokens = nltk.word_tokenize(filtered_sentence)
            single_tokens = list(set(tokens))

            for token in single_tokens:
                if not self.gr.has_node(token):
                    self.gr.add_node(str(token))

            for i, token in enumerate(tokens):
                if i != 0:
                    edge = (tokens[i - 1], token)
                    if not self.gr.has_edge(edge):
                        self.gr.add_edge(edge,
                                         wt=1.0,
                                         label=START_OCCURRENCES_NUM)
                    else:
                        #If the edge exists, the weight of the edge shall divided by the number of occurrences of the couple of terms.
                        #Therefore, we shall keep memory of the number of occurrences for each couple of terms.
                        number_of_occurrences = self.gr.edge_label(edge)
                        new_number_of_occurrences = number_of_occurrences + 1
                        self.gr.set_edge_label(edge, new_number_of_occurrences)
                        self.gr.set_edge_weight(edge,
                                                wt=1.0 /
                                                new_number_of_occurrences)
 def createNetFromSentences(self, sentences):
     "This function creates the network starting from a set of sentences"
     
     text_filter = TextFilter()
     for sentence in sentences:
         filtered_sentence = text_filter.filter_all(sentence)
         
         
         tokens = nltk.word_tokenize(filtered_sentence)
         single_tokens = list(set(tokens))
         
         for token in single_tokens:
             if not self.gr.has_node(token):
                 self.gr.add_node(str(token))
         
         for i, token in enumerate(tokens):
             if i != 0:
                 edge = (tokens[i-1], token)
                 if not self.gr.has_edge(edge):
                     self.gr.add_edge(edge, wt=1.0, label = START_OCCURRENCES_NUM)
                 else:
                     #If the edge exists, the weight of the edge shall divided by the number of occurrences of the couple of terms.
                     #Therefore, we shall keep memory of the number of occurrences for each couple of terms.
                     number_of_occurrences = self.gr.edge_label(edge)
                     new_number_of_occurrences = number_of_occurrences + 1
                     self.gr.set_edge_label(edge, new_number_of_occurrences)
                     self.gr.set_edge_weight(edge, wt = 1.0/new_number_of_occurrences)
Ejemplo n.º 3
0
 def __init__(self, modelID, inputXMLfilepath= "", modelType="", title="", objects=[]):
     '''
     Constructor
     @param modelID: identifier of the model
     @param inputXMLfilepath: path to the input XML file containing the model 
     if this parameter is left empty a new XML tree is created
     @param type: KAOS, TROPOS, or any other kind of model
     '''
     self.textFilter = TextFilter()
     self.wordTokenizer = TreebankWordTokenizer()        
     self.maxID = "100"  #@todo: we have to set the current maximum to the actual maximum value
                         #for the model
     self.modelInfo = ModelInfo(modelID)
     
     if not inputXMLfilepath == "":
         
         self.modelInfo.setLocation(inputXMLfilepath)
         
         self.tree =  ET.parse(self.modelInfo.getLocation())
     
         self.__loadModelInfo(self.modelInfo)
         self.modelGoals = self.__loadModelGoals()
         self.modelWords = self.__loadModelWords()
         self.modelStems = self.__loadModelStems()
     else:
         attributes = dict()
         attributes['type'] = modelType
         attributes['title'] = title
         attributes['object'] = objects
         root = Element("MODEL", attributes)
         self.tree = ElementTree(root)
Ejemplo n.º 4
0
 def __init__(self, modelIndexManager):
     '''
     @param modelIndex: reference to the place where the models are indexed
     '''
     self.textFilter = TextFilter()
     self.modelIndexManager = modelIndexManager
     self.wordTokenizer = TreebankWordTokenizer()
     self.tRecommender = TransformationRecommender() 
class TransformationRecommender(object):
    '''
    This class recommends a transformation according
    to the model information ModelInfo object and the query issued
    '''
    
    '''
    This object is a Singleton, since it does not have private data
    but only functions: the code below defines a singleton
    '''
    _instance = None
    
    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(TransformationRecommender, cls).__new__(
                                cls, *args, **kwargs)
        return cls._instance
    
    def __init__(self):
        self.tf = TextFilter()
        self.wordTokenizer = TreebankWordTokenizer()

    def getRecommendedTransformation(self, modelInfo, query):
        '''
        If the input sentence is the same as the title except than in the title
        part that specifies the object, then "object change" shall be suggested
        '''
        title = modelInfo.getName()
        titleFiltered = self.tf.filter_all_except_stem(title)
        titleToks = self.wordTokenizer.tokenize(titleFiltered)
        
        titleToksNoObj = [t for t in titleToks if t not in modelInfo.getObjects()]
        
        queryFiltered = self.tf.filter_all_except_stem(query)
        sentenceToks = self.wordTokenizer.tokenize(queryFiltered)
        
        if set(titleToksNoObj).issubset(sentenceToks):
            return OBJECT_CHANGE
        else:
            return ''
sentenceNetGediminas.write_graph('gediminas_graph.gv')
print 'Gediminas size', len(sentenceNetGediminas.get_net().nodes())

sentenceNetRicci = SentenceNetCreator()
sentenceNetRicci.createNet([fp3])
sentenceNetRicci.write_graph('ricci_graph.gv')
print 'Ricci size', len(sentenceNetRicci.get_net().nodes())

sentenceNetWikiGedi = SentenceNetCreator()
sentenceNetWikiGedi.createNet([fp4])
sentenceNetWikiGedi.write_graph('wikigedi_graph.gv')
print 'WikiGedi size', len(sentenceNetWikiGedi.get_net().nodes())

print "Weighted Knowledge Graphs created"

terms_filter = TextFilter()
sentence = "The system shall display similar books"
filtered_sent = terms_filter.filter_all(sentence)

visitor_wiki = SentenceNetVisitor(sentenceNetWiki.get_net(), sentenceNetWiki.get_edge_start_weight(), sentenceNetWiki.get_start_occurrences_num())
path_wiki, path_weight_wiki = visitor_wiki.search_A_star(filtered_sent)

print path_wiki
print path_weight_wiki

visitor_gediminas = SentenceNetVisitor(sentenceNetGediminas.get_net(), sentenceNetGediminas.get_edge_start_weight(), sentenceNetGediminas.get_start_occurrences_num())
path_gediminas, path_weight_gediminas = visitor_gediminas.search_A_star(filtered_sent)

print path_gediminas
print path_weight_gediminas
from SentenceNetVisitor import SentenceNetVisitor
from XMLReqManager import XMLReqManager
from SentenceNetCreator import SentenceNetCreator
from irutils.TextFilter import TextFilter

s1 = SentenceNetCreator()
n1 = s1.get_net()
v1 = SentenceNetVisitor(n1, s1.get_edge_start_weight(),
                        s1.get_start_occurrences_num())

xml_doc_handler = XMLReqManager('req_document.xsd', '2007 - eirene fun 7.xml')
req_document = xml_doc_handler.get_requirements_text()

terms_filter = TextFilter()

for sent in req_document:
    filtered_sent = terms_filter.filter_all(sent)
    path1, path_weight1 = v1.search_A_star(filtered_sent)

print 'now producing a random sentence according to the document learnt...'
print v1.get_random_sentence('network', 100)
Ejemplo n.º 8
0
class RequirementsModel(object):
    '''
    This class embeds the information residing in the XML
    of a requirements model passed as input parameter
    during construction 
    '''

    def __init__(self, modelID, inputXMLfilepath= "", modelType="", title="", objects=[]):
        '''
        Constructor
        @param modelID: identifier of the model
        @param inputXMLfilepath: path to the input XML file containing the model 
        if this parameter is left empty a new XML tree is created
        @param type: KAOS, TROPOS, or any other kind of model
        '''
        self.textFilter = TextFilter()
        self.wordTokenizer = TreebankWordTokenizer()        
        self.maxID = "100"  #@todo: we have to set the current maximum to the actual maximum value
                            #for the model
        self.modelInfo = ModelInfo(modelID)
        
        if not inputXMLfilepath == "":
            
            self.modelInfo.setLocation(inputXMLfilepath)
            
            self.tree =  ET.parse(self.modelInfo.getLocation())
        
            self.__loadModelInfo(self.modelInfo)
            self.modelGoals = self.__loadModelGoals()
            self.modelWords = self.__loadModelWords()
            self.modelStems = self.__loadModelStems()
        else:
            attributes = dict()
            attributes['type'] = modelType
            attributes['title'] = title
            attributes['object'] = objects
            root = Element("MODEL", attributes)
            self.tree = ElementTree(root)
    
    def __loadModelInfo(self, modelInfo):
        '''
        This function load the name of the model from the "title" field of the MODEL tag,
        together with the type and the objects, and stores these information in the 
        ModelInfo object
        '''
        root = self.tree.getroot()
        
        modelInfo.setName(self.textFilter.lower_all(root.get("title")))
        modelInfo.setType(self.textFilter.lower_all(root.get("type")))
        
        objects = root.get("object").strip().split(OBJ_SEPARATOR)
        lowercaseObjects = [self.textFilter.lower_all(o) for o in objects]
        modelInfo.setObjects(lowercaseObjects)   
    
    
    def __loadModelGoals(self):
        '''
        The function loads the goal names included in the model
        and returns a list with all the goals of the model.
        The goals names are stored as lowercase goals
        '''  
        root = self.tree.getroot()
        goalNames = list()

        for child in root.iter('ENTITY'):
            if child.attrib['type'] == 'goal': 
                goalNames.append(self.textFilter.lower_all(child.attrib['name'])) 
                    
        return goalNames
        
        
    def __loadModelWords(self):
        '''
        The function loads the words included in the model
        and returns a dictionary with all the words of the model
        and their frequency
        '''
               
        tokenizedWords = dict()

        if not self.modelGoals == None:
            for name in self.modelGoals:
                nameFiltered = self.textFilter.filter_all_except_stem(name)
                words = self.wordTokenizer.tokenize(nameFiltered)
                for word in words:
                    if not tokenizedWords.has_key(word): 
                        tokenizedWords[word] = 1
                    else:
                        tokenizedWords[word] = tokenizedWords[word] + 1
                    
        return tokenizedWords
        
    def __loadModelStems(self):
        '''
        The function loads the stems included in the model
        and returns a dictionary with all the stems of the model
        and their frequency
        ''' 
        tokenizedStems = dict()
        
        if not self.modelWords == None:
            for w in self.modelWords.keys():
                stem = self.textFilter.filter_all(w)
                if not tokenizedStems.has_key(stem):
                    tokenizedStems[stem] = self.modelWords[w]
                else:
                    tokenizedStems[stem] = tokenizedStems[stem] + self.modelWords[w]
                    
        return tokenizedStems
        
    def __getModelStems(self):
        return self.modelStems.keys()
    
    def __getModelWords(self):
        return self.modelWords.keys()
    
    def __getModelGoals(self):
        return self.modelGoals
    
    def __getModelStemsAndFreq(self):
        return self.modelStems
    
    def __getModelWordsAndFreq(self):
        return self.modelWords
    
    def getModelInfo(self):
        return self.modelInfo
    
    def getModelID(self):
        return self.modelInfo.getId()
    
    def getModelKeys(self, keyType):
        if keyType == STEM_STRING:
            return self.__getModelStems()
        if keyType == WORD_STRING:
            return self.__getModelWords()
        if keyType == GOAL_STRING:
            return self.__getModelGoals() 
        
    def getModelKeysAndFrequencies(self, keyType):
        if keyType == STEM_STRING:
            return self.__getModelStemsAndFreq()
        if keyType == WORD_STRING:
            return self.__getModelWordsAndFreq()
        if keyType == GOAL_STRING:
            return dict(zip(self.__getModelGoals()), [1] * (len(self.__getModelGoals())) )
            
        
    def changeTitle(self, newTitle):
        '''
        This function shall change the title of the model, 
        which means changing the modelInfo and the XML
        of the model
        '''
        #self.modelInfo.setName(newTitle)
        
        root = self.tree.getroot()
        root.set("title", newTitle)
        self.__loadModelInfo(self.modelInfo) #the function updates the modelInfo structure
    
    def changeObjects(self, newObjectsList):
        '''
        This function shall change the objects of the model,
        which means changing the modelInfo 
        but also the XML of the model
        '''
        
        root = self.tree.getroot()
        
        newObjects = ' ,'.join([o for o in newObjectsList])
        root.set("object", newObjects)  
        self.__loadModelInfo(self.modelInfo)
        
    def changeGoalName(self, goalID, newGoalName):
        '''
        @param goalID: ID of the goal that shall have a new name
        @param newGoalName: string representing the new name of the goal  
        '''
        root = self.tree.getroot()

        for child in root.iter('ENTITY'):
            if child.attrib['type'] == 'goal' and child.attrib['id'] == goalID:
                child.attrib['name'] = newGoalName 
        
    def searchGoalByName(self, goalName):
        '''
        @param goalName: name of the goal to be searched
        return: goalID, which is the unique ID of the goal, if the goal exist
                -1, if the goal is not found
        '''
        root = self.tree.getroot()

        for child in root.iter('ENTITY'):
            if child.attrib['type'] == 'goal' and child.attrib['name'] == goalName:
                return child.attrib['id']
        
        return -1 
    
    def searchGoalsBySubstring(self, goalSubstring, caseSensitive = "NO"):
        '''
        @param goalSubstring: a substring that shall be searched among the goal names. 
        By default the search is not case sensitive
        return: a list with the couples [ID, goalName] of the goals that include the @param goalSubstring
        '''
        root = self.tree.getroot()
        goalDict = dict()

        for child in root.iter('ENTITY'):
            if child.attrib['type'] == 'goal': 
                if caseSensitive == "NO":
                    if self.textFilter.lower_all(goalSubstring) in self.textFilter.lower_all(child.attrib['name']):
                        goalDict[child.attrib['id']] = child.attrib['name']
                else:
                    if goalSubstring in child.attrib['name']:
                        goalDict[child.attrib['id']] = child.attrib['name']
                
        
        return goalDict
    
    def __assignUniqueIDs(self, treeRoot):
        '''
        This function assigns unique IDs to all the objects 
        of type ENTITY in @param tree
        '''
        currentMaxId = self.maxID
        for child in treeRoot.iter('ENTITY'):
            currentMaxId = str( int(currentMaxId) + 1 )
            child.attrib['id'] = currentMaxId
            
        self.maxID = currentMaxId
    
    def insertTree(self, parentID, childTree):
        '''
        Given a @param childTree, which is a tree or a node, this is added as a child of parentID
        below the first refinement of the parent. 
        The assumption here is that each parent can have ONLY ONE TYPE of refinement.
        The unique IDs to the child elements are dynamically assigned by the function. 
        The childTree could be also a single node.
        '''
        root = self.tree.getroot()
        
        for child in root.iter('ENTITY'):
            if child.attrib['id'] == parentID:
                refinement = child.findall("REFINEMENT")
                if refinement and len(refinement) == 1: #ONLY ONE TYPE of refinement is allowed for each element
                    self.__assignUniqueIDs(childTree)
                    refinement[0].append(childTree)
                    return

    def saveModelAs(self, destinationFilePath):
        '''
        @param destinationFilePath: path of the file where the model shall be saved.
        @todo: currently the model is saved to another location and the original location
        is lost. Therefore, the model currently keeps the same ID. We have to change
        this behaviour. 
        '''
        self.modelInfo.setLocation(destinationFilePath) 
        self.saveModel()
        
    def saveModel(self):
        '''
        Save the model in the same destination as the input folder
        and with the original name
        '''
        try:
            self.tree.write(self.modelInfo.getLocation())
        except IOError:
            print "IOError: Saving to a path that does not exist! Use saveModelAs() instead"
        except:
            print "An error occurred"
Ejemplo n.º 9
0
class QueryManager(object):
    '''
    Given a specification query, this object returns a set of models
    together with possible transformations that can be applied to the model
    to address the satisfy the specification query
    '''


    def __init__(self, modelIndexManager):
        '''
        @param modelIndex: reference to the place where the models are indexed
        '''
        self.textFilter = TextFilter()
        self.modelIndexManager = modelIndexManager
        self.wordTokenizer = TreebankWordTokenizer()
        self.tRecommender = TransformationRecommender() 
        
    def __parseQuery(self, queryString):
        '''
        This function returns the words included in queryString,
        after filtering all the stopwords, performing stemmming
        and applying all the filters provided by textFilter
        @param queryString: the specification query in the form of a string 
        ''' 
        filteredQueryString = self.textFilter.filter_all(queryString)
        return self.wordTokenizer.tokenize(filteredQueryString)
        
    def issueQuery(self, queryString):
        '''
        This is the main function of this class. Given the specification
        query, the function parses the specification and returns a
        set of QueryResult objects, which include the link to the models
        @param queryString: the specification query in the form of a string
        @return: a list of QueryResult objects.
        '''
        qr = list()
        
        stems = self.__parseQuery(queryString)
        for stem in stems:
            
            modelsInfos = self.modelIndexManager.searchModels(stem, STEM_STRING)
            
            #modelsTransformationsList = [(model, "object change") for model in models]
            #results[stem] = modelsTransformationsList
        
            if not modelsInfos == None:
                for modelInfo in modelsInfos:
                    score = 0.1
                    transformation = self.tRecommender.getRecommendedTransformation(modelInfo, queryString)
                    qr.append(QueryResult(modelInfo, [transformation], score))
                    
            qr.sort(key=lambda x: x.score) #the list is ordered by the score attribute and reversed
            qr.reverse()
        
        '''
        @todo: for each model we shall understand which is the best transformation.
        To this end, an additional class is required.
        Currently, we always add the object change transformation together 
        with each model found. 
        '''
        
        return qr   
Ejemplo n.º 10
0
from SentenceNetVisitor import SentenceNetVisitor
from XMLReqManager import XMLReqManager
from SentenceNetCreator import SentenceNetCreator
from irutils.TextFilter import TextFilter

s1 = SentenceNetCreator()
n1 = s1.get_net()
v1 = SentenceNetVisitor(n1, s1.get_edge_start_weight(),
                        s1.get_start_occurrences_num())

xml_doc_handler = XMLReqManager('req_document.xsd', '2007 - eirene fun 7.xml')
req_document = xml_doc_handler.get_requirements_text()

terms_filter = TextFilter()

for sent in req_document:
    filtered_sent = terms_filter.filter_all(sent)
    filtered_sent = terms_filter.remove_item(filtered_sent, "\"")
    filtered_sent = terms_filter.remove_item(filtered_sent, "-")
    print filtered_sent
    v1.search_A_star(filtered_sent)

s1.write_graph("eireneGraph.gv")
from SentenceNetVisitor import SentenceNetVisitor
from XMLReqManager import XMLReqManager
from SentenceNetCreator import SentenceNetCreator
from irutils.TextFilter import TextFilter


s1 = SentenceNetCreator()
n1 = s1.get_net()
v1 = SentenceNetVisitor(n1, s1.get_edge_start_weight(), s1.get_start_occurrences_num())

xml_doc_handler = XMLReqManager('req_document.xsd', '2007 - eirene fun 7.xml')
req_document = xml_doc_handler.get_requirements_text()

terms_filter = TextFilter()

for sent in req_document:
    filtered_sent = terms_filter.filter_all(sent)
    filtered_sent = terms_filter.remove_item(filtered_sent, "\"")
    filtered_sent = terms_filter.remove_item(filtered_sent, "-")
    print filtered_sent
    v1.search_A_star(filtered_sent)

s1.write_graph("eireneGraph.gv")
 def __init__(self):
     self.tf = TextFilter()
     self.wordTokenizer = TreebankWordTokenizer()