예제 #1
0
 def getIndicativeSentences(self,topK,intersectionTh):
     if len(self.indicativeSentences) > 0:
         return self.indicativeSentences
     else:
         topToksTuples = self.indicativeWords[:topK]
         topToks = [k for k,_ in topToksTuples]
         
         for d in self.documents:
             sents = d.getSentences()
             self.sentences.extend(sents)
         
         impSents ={}
         for sent in self.sentences:
             if sent not in impSents:
                 sentToks = utils.getTokens(sent)
                 if len(sentToks) > 100:
                     continue
                 intersect = utils.getIntersection(topToks, sentToks)
                 if len(intersect) > intersectionTh:
                     impSents[sent] = len(intersect)
                     #if sent not in impSentsF:
                     #    impSentsF[sent] = len(intersect)
                 #allImptSents.append(impSents)
         
         self.indicativeSentences = utils.getSorted(impSents.items(),1)
         return self.indicativeSentences
예제 #2
0
 def getIndicativeSentences(self,topK,intersectionTh):
     if len(self.indicativeSentences) > 0:
         return self.indicativeSentences
     else:
         topToksTuples = self.indicativeWords[:topK]
         #topToksTuples = self.indicativeWords
         topToks = [k for k,_ in topToksTuples]
         
         for d in self.documents:
             sents = d.getSentences()
             if sents and len(sents)>0:
                 self.sentences.extend(sents)
         
         impSents ={}
         for sent in self.sentences:
             if sent not in impSents:
                 sentToks = utils.getTokens(sent)
                 if len(sentToks) > 100:
                     continue
                 intersect = utils.getIntersection(topToks, sentToks)
                 if len(intersect) > intersectionTh:
                     #impSents[sent] = len(intersect)
                     impSents[sent] = intersect
                     #print intersect
                     #if sent not in impSentsF:
                     #    impSentsF[sent] = len(intersect)
                 #allImptSents.append(impSents)
         if impSents:
             #self.indicativeSentences = utils.getSorted(impSents.items(),1)
             self.indicativeSentences = sorted(impSents.items(),key=lambda x: len(x[1]), reverse=True)
             #sortedToksTFDF = sorted(toksTFDF.items(), key=lambda x: x[1][0], reverse=True)
         return self.indicativeSentences
예제 #3
0
 def extractWebpageEventModel(self, text):
     webpageEventModel = {}
     
     entities = self.webpageEntities(text)
     if len(entities) > 1:
         for sent in entities:
             dictval = sent[1]
             for k in dictval:
                 if k in ["LOCATION","Disaster","DATE"]:
                     if webpageEventModel.has_key(k):
                         webpageEventModel[k].extend(dictval[k])
                     else:
                         webpageEventModel[k] = []
                         webpageEventModel[k].extend(dictval[k])
         for k in webpageEventModel:
             if k in ["LOCATION","DATE"]:
                 webpageEventModel[k] = dict(self.getEntitiesFreq(webpageEventModel[k]))
         
         webpageToks = eventUtils.getTokens(text)
         webpageDis = set(webpageEventModel['Disaster'])
         webpageDisDic = {}
         for wd in webpageDis:
             webpageDisDic[wd]=webpageToks.count(wd)
         webpageEventModel['Disaster']=webpageDisDic
         
         
     return webpageEventModel
예제 #4
0
 def getWords(self):
     if self.words:
         return self.words
     else:
         r = utils.getTokens(self.text)
         if r:
             self.words = r
             return self.words
예제 #5
0
 def calculate_similarity_equalWeights_duplicate(self,doc):
     eDisDic = self.entities['Topic']
     
     locToks = self.entities['LOCATION'].keys()
     locToks = eventUtils.getStemmedWords(locToks)
     locDic = dict(zip(locToks,self.entities['LOCATION'].values()))
     
     dToks = self.entities['DATE'].keys()
     dToks = eventUtils.getStemmedWords(dToks)
     dDic = dict(zip(dToks,self.entities['DATE'].values()))
     
     tokens = eventUtils.getTokens(doc)
     tokensDic = eventUtils.getFreq(tokens)
     wv = [1+math.log(e) for e in tokensDic.values()]
     wvScalar = self.getScalar(wv)
     scores = []
     
     ksd = 0    
     #interst = 0
     for i in tokensDic:
         if i in eDisDic:
             ksd += (1+math.log(eDisDic[i]))* (1+math.log(tokensDic[i]))
             #interst +=1
     if ksd > 0:
         ksd = float(ksd)/(self.scalars['Topic'] * wvScalar)
     else:
         ksd = 0
     if ksd == 0:
         return 0
     #if interst < 2:
         #return 0
     scores.append(ksd)
     ksl = 0    
     for i in tokensDic:
         if i in locDic:
             ksl += (1+math.log(locDic[i]))* (1+math.log(tokensDic[i]))
     if ksl > 0:
         ksl = float(ksl)/(self.scalars['LOCATION'] * wvScalar)
         
     else:
         ksl = 0
     scores.append(ksl)
     
     ks = 0    
     for i in tokensDic:
         if i in dDic:
             ks += (1+math.log(dDic[i]))* (1+math.log(tokensDic[i]))
     if ks > 0:
         ks = float(ks)/(self.scalars['DATE'] * wvScalar)
         
     else:
         ks = 0
     scores.append(ks)
     
     score = sum(scores) / 3.0
     return score
예제 #6
0
 def getWords(self):
     if self.words:
         return self.words
     else:
         r = utils.getTokens(self.text)
         if len(r)>0:
             self.words = [w for w in r]
             return self.words
         else:
             return []
예제 #7
0
 def calculate_score(self, doc):
     #sims=[]
     docWords = getTokens(doc)
     docTF = getFreq(docWords)
     sim = self.cosSim( docTF)
     
     if sim >= self.relevanceth:
         return [1,sim]
     else:
         return [0,sim]
예제 #8
0
 def calculate_similarity(self,doc):
     '''
     eDisDic = self.probEvtModel['Topic']
     
     if self.locDic ==[]:
         locToks = self.probEvtModel['LOCATION'].keys()
         locToks = eventUtils.getStemmedWords(locToks)
         self.locDic = dict(zip(locToks,self.probEvtModel['LOCATION'].values()))
     
     if self.dDic == []:
         dToks = self.probEvtModel['DATE'].keys()
         dToks = eventUtils.getStemmedWords(dToks)
         self.dDic = dict(zip(dToks,self.probEvtModel['DATE'].values()))
     '''
     tokens = eventUtils.getTokens(doc)
     docProb = {}
     #tokensDic = eventUtils.getFreq(tokens)
     #wv = [1+math.log(e) for e in tokensDic.values()]
     docProb['Topic'] = {}
     total = 0.0
     for t in tokens:
         if t in self.eDisDic:
             p = self.eDisDic[t]
             total = total + math.log(p)
     if total == 0.0:
         return -100
     docProb['Topic']['Total'] = total
     
     docProb['LOCATION'] = {}
     total = 0.0
     for t in tokens:
         if t in self.locDic:
             p = self.locDic[t]
             total = total + math.log(p)
     docProb['LOCATION']['Total'] = total
     
     docProb['DATE'] = {}
     total = 0.0
     for t in tokens:
         if t in self.dDic:
             p = self.dDic[t]
             total = total + math.log(p)
     docProb['DATE']['Total'] = total
     
         
     #finalDocProb = 1
     finalDocProb = 0.0
     for k in docProb:
         #finalDocProb = finalDocProb * docProb[k]['Total']
         finalDocProb = finalDocProb + docProb[k]['Total']
     docProb['Total'] = finalDocProb
     #if finalDocProb == 0.0:
     #    finalDocProb = -100.0
     return finalDocProb*-1
예제 #9
0
    def calculate_similarity(self, doc):
        eDisDic = self.entities["Disaster"]

        locToks = self.entities["LOCATION"].keys()
        locToks = eventUtils.getStemmedWords(locToks)
        locDic = dict(zip(locToks, self.entities["LOCATION"].values()))

        dToks = self.entities["DATE"].keys()
        dToks = eventUtils.getStemmedWords(dToks)
        dDic = dict(zip(dToks, self.entities["DATE"].values()))

        tokens = eventUtils.getTokens(doc)
        tokensDic = eventUtils.getFreq(tokens)
        wv = [1 + math.log(e) for e in tokensDic.values()]
        wvScalar = self.getScalar(wv)
        scores = []

        ksd = 0
        for i in tokensDic:
            if i in eDisDic:
                ksd += (1 + math.log(eDisDic[i])) * (1 + math.log(tokensDic[i]))
        if ksd > 0:
            ksd = float(ksd) / (self.scalars["Disaster"] * wvScalar)
        else:
            ksd = 0
        if ksd == 0:
            return 0
        scores.append(ksd)
        ksl = 0
        for i in tokensDic:
            if i in locDic:
                ksl += (1 + math.log(locDic[i])) * (1 + math.log(tokensDic[i]))
        if ksl > 0:
            ksl = float(ksl) / (self.scalars["LOCATION"] * wvScalar)

        else:
            ksl = 0
        scores.append(ksl)

        ks = 0
        for i in tokensDic:
            if i in dDic:
                ks += (1 + math.log(dDic[i])) * (1 + math.log(tokensDic[i]))
        if ks > 0:
            ks = float(ks) / (self.scalars["DATE"] * wvScalar)

        else:
            ks = 0
        scores.append(ks)

        score = sum(scores) / 3.0
        return score
예제 #10
0
 def calculate_score(self,doc,m):
     
     #docScore = 0.0
     
     
     if m == 'W':
         docEnt = eventUtils.getEntities(doc)[0]
         docEnt['Topic'] = eventUtils.getTokens(doc)
         score = self.getDocProb(docEnt)
     else:
         
         score = self.calculate_similarity(doc)
     return score
예제 #11
0
 def calculate_score_AllDocs(self, doc):
     sims=[]
     docWords = getTokens(doc)
     docTF = getFreq(docWords)
     ndocTF = dict.fromkeys(self.topVocabDic)
     for k in ndocTF:
         if k in docTF:
             ndocTF[k] = docTF[k]
         else:
             ndocTF[k] = 1/math.e
     for dTF in self.docsTF:
         s = self.cosSim(ndocTF, dTF)
         sims.append(s)
     sim = max(sims)
     if sim >= self.relevanceth:
         return [1,sim]
     else:
         return [0,sim]
예제 #12
0
 def webpageEntities(self,docText=""):
     disasters=set(self.entities["Disaster"].keys())
     
     sentences = eventUtils.getSentences(docText)
     webpageEnts =[]
     for sent in sentences:
         sentToks = eventUtils.getTokens(sent)
         if len(sentToks) > 100:
             continue
         intersect = eventUtils.getIntersection(disasters, sentToks)
         if len(intersect) > self.intersectionTh:
             #print intersect
             sentEnts = eventUtils.getEntities(sent)[0]
             if sentEnts.has_key('LOCATION') or sentEnts.has_key('DATE'):
                 sentEnts['Disaster'] = intersect
                 webpageEnts.append((sent,sentEnts))
     
     return webpageEnts
예제 #13
0
def getEM_Sents(wps):
    docsEntities=[]
    docsEntitiesFreq = []
    entitiesProb = {}
    
    
    collSents = []
    #for i,wp in enumerate(wps):
    for wp in wps:
        if 'text' not in wp:
            continue
        wpContent = wp['text']+wp['title']
        wpSplit = wpContent.split('\n')
        wpFiltered = filter(None,wpSplit)
        wpContentf = '\n'.join(wpFiltered)
        sents = eventUtils.getSentences(wpContentf)
        collSents.append(sents)
    allSents = []
    for sents in collSents:
        allSents.extend(sents)
    fw = eventUtils.getFreqTokens(allSents)
    fw = [w[0] for w in fw]
    
    #collFilteredSents = []
    collEventModelInsts=[]
    for sents in collSents:
        filtEvtModelInsts = []
        for s in sents:
            sentToks = eventUtils.getTokens(s)
            cw = eventUtils.getIntersection(fw, sentToks)
            if len(cw) >= 2:
                emi = {}
                emi['TOPIC'] = list(cw)
                ents = eventUtils.getEntities(s)[0]
                if ents.has_key('LOCATION'):
                    emi['LOCATION'] = ents['LOCATION']
                    #filtEvtModelInsts.append(emi)
                if ents.has_key('DATE'):
                        #emi['TOPIC'] = cw
                    emi['DATE']=ents['DATE']
                filtEvtModelInsts.append(emi)
        collEventModelInsts.append(filtEvtModelInsts)
    '''
예제 #14
0
 def calculate_similarity_intersect(self,doc):
     #tokens = getTokenizedDoc(doc)
     tokens = eventUtils.getTokens(doc)
     doc_set = set(tokens)
     
     scores = []
     
     for k in self.entities:
         entSet = set(self.entities[k].keys())
         intersect = len(doc_set & entSet)
         union = len(doc_set | entSet)
         if k == "Disaster":
             if intersect == 0:
                 return 0
         
         score = intersect * 1.0 / union #len(self.entities[k])
         
         scores.append(score)
     
     score = sum(scores)/3.0     
     return score
예제 #15
0
 def webpageEntities_old(self,docText=""):
     disasters=self.entities["Disaster"]
     
     sentences = eventUtils.getSentences(docText)
     #impSentences = getIndicativeSents(sentences, disasters, len(disasters), 0)
     #impSentences = []
     webpageEnts =[]
     for sent in sentences:
         sentToks = eventUtils.getTokens(sent)
         if len(sentToks) > 100:
             continue
         intersect = eventUtils.getIntersection(disasters, sentToks)
         if len(intersect) > self.intersectionTh:
             #impSentences.append(sent)
             sentEnts = eventUtils.getEntities(sent)[0]
             if sentEnts.has_key('LOCATION') or sentEnts.has_key('DATE'):
                 sentEnts['Disaster'] = intersect
                 webpageEnts.append((sent,sentEnts))
     #entities = getEntities(impSentences)
     #webpageEnts = zip(impSentences,entities)
     
     return webpageEnts
예제 #16
0
def buildProbEventModel(docsList):
    t = ''
    docsTotalFreqs=[]
    docsEntities=[]
    docsEntitiesFreq = []
    entitiesProb = {}
    
    # Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
    # Also produces the total frequency for each document of each list (tokens, locations, and dates)
    for doc in docsList:
        
        if doc.has_key('text'):
            t = doc['text']
            if doc.has_key('title'):
                t =doc['title']+ " "+t
        if t:
            print 'Reading ' + t[:100]
            ents = eventUtils.getEntities(t)[0]
            docEnt = {}
            docEnt['LOCATION']={}
            if 'LOCATION' in ents:
                docEnt['LOCATION'] =  ents['LOCATION']
            docEnt['DATE']={}
            if 'DATE' in ents:
                docEnt['DATE'] = ents['DATE']
            toks = eventUtils.getTokens(t)
            docEnt['Topic'] = toks
            docsEntities.append(docEnt)
            
            docEntFreq = {}
            #docTotals = {}
            for k in docEnt:
                docEntFreq[k] = eventUtils.getFreq(docEnt[k])
                #totalFreq = sum([v for _,v in docEntFreq[k].items()])
                
                #docTotals[k] = totalFreq
            docsEntitiesFreq.append(docEntFreq)
            #docsTotalFreqs.append(docTotals)
    
    # Collection-level frequency for each entity(tokens, locations, dates)
    
    # Total Frequency of keywords, locations, and dates in all documents
    '''
    allDocsTotal = {}
    allDocsTotal['LOCATION'] = 0
    allDocsTotal['DATE']=0
    allDocsTotal['Topic'] = 0
    
    for docTotFreq in docsTotalFreqs:
        for k in docTotFreq:
            allDocsTotal[k]+= docTotFreq[k]
    '''
    
    #Calculating prob for each item in each entity lists (tokens, locations, and dates) as 
    # freq of item in all docs / total freq of all terms in that list
    entitiesProb['LOCATION']={}
    entitiesProb['DATE']={}
    entitiesProb['Topic']={}
    
    for docEntFreq in docsEntitiesFreq:
        for entity in docEntFreq:
            for val in docEntFreq[entity]:
                if val in entitiesProb[entity]:
                    entitiesProb[entity][val] += docEntFreq[entity][val]
                else:
                    entitiesProb[entity][val] = docEntFreq[entity][val]
    
    for ent in entitiesProb:
        allvalsFreq = sum([v for _,v in entitiesProb[ent].items()])
        for k in entitiesProb[ent]:
            #entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])
            
            entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (len(entitiesProb[ent]) + allvalsFreq)
            
        
            
    return docsEntities, entitiesProb
예제 #17
0
    def buildEventModel_old(self, seedURLs):

        corpus = Collection(seedURLs)
        #sortedTokensFreqs = corpus.getWordsFrequencies()
        sortedToksTFDF = corpus.getIndicativeWords()
        print sortedToksTFDF
        sortedImptSents = corpus.getIndicativeSentences(
            self.topK, self.intersectionTh)
        # Get Event Model
        eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)
        #topToks = [k for k,_ in sortedToksTFDF]
        #if self.topK < len(topToks):
        #    topToks =  topToks[:self.topK]
        #self.entities['Disaster'] = set(topToks)

        self.entities['LOCATION'] = []
        self.entities['DATE'] = []
        for e in eventModelInstances:
            if 'LOCATION' in e:
                self.entities['LOCATION'].extend(e['LOCATION'])
            elif 'DATE' in e:
                self.entities['DATE'].extend(e['DATE'])

        entitiesFreq = {}
        entitiesFreq['LOCATION'] = eventUtils.getFreq(
            self.entities['LOCATION'])
        entitiesFreq['LOCATION'] = eventUtils.getSorted(
            entitiesFreq['LOCATION'].items(), 1)
        entitiesFreq['DATE'] = eventUtils.getFreq(self.entities['DATE'])
        entitiesFreq['DATE'] = eventUtils.getSorted(
            entitiesFreq['DATE'].items(), 1)

        l = [k for k, _ in entitiesFreq['LOCATION']]
        if self.topK < len(l):
            #l = l[:self.topK]
            l = l[:3]
        self.entities['LOCATION'] = set(l)

        d = [k for k, _ in entitiesFreq['DATE']]
        if self.topK < len(d):
            #d = d[:self.topK]
            d = d[:3]
        self.entities['DATE'] = set(d)
        '''
        locList = self.entities['LOCATION']
        locSet = set(locList)
        self.entities['LOCATION'] = [l for l in locSet]
        '''
        self.entities['LOCATION'] = self.getUniqueEntities(
            self.entities['LOCATION'])
        '''
        dateList = self.entities['DATE']
        dateSet = set(dateList)
        self.entities['DATE'] = [d for d in dateSet]
        '''
        self.entities['DATE'] = self.getUniqueEntities(self.entities['DATE'])

        locDate = list(self.entities['LOCATION']) + list(self.entities['DATE'])
        locDate = eventUtils.getTokens(' '.join(locDate))

        ntopToks = []
        topToks = [k for k, _ in sortedToksTFDF]
        for tok in topToks:
            if tok not in locDate:
                ntopToks.append(tok)
        topToks = ntopToks
        if self.topK < len(topToks):
            topToks = topToks[:self.topK]
        self.entities['Disaster'] = set(topToks)

        self.allEntities = []
        for k in self.entities:
            self.allEntities.extend(self.entities[k])

        print self.allEntities
예제 #18
0
    def buildEventModel(self, keywordsTh, seedURLs):

        corpus = Collection(seedURLs)

        #NoTFDF
        sortedToksTFDF = corpus.getIndicativeWords()
        self.toksTFDFDic = dict(sortedToksTFDF)
        #print sortedToksTFDF

        #sortedImptSents = corpus.getIndicativeSentences(self.topK,self.intersectionTh)
        sortedImptSents = corpus.getIndicativeSentences(
            keywordsTh, self.intersectionTh)
        # Get Event Model
        eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)

        self.entities['LOCATION'] = []
        self.entities['DATE'] = []
        self.entities['Disaster'] = []
        for e in eventModelInstances:
            if 'LOCATION' in e:
                self.entities['LOCATION'].extend(e['LOCATION'])
            elif 'DATE' in e:
                self.entities['DATE'].extend(e['DATE'])
            self.entities['Disaster'].extend(e['Disaster'])

        entitiesFreq = {}
        entitiesFreq['LOCATION'] = self.getEntitiesFreq(
            self.entities['LOCATION'])
        entitiesFreq['DATE'] = self.getEntitiesFreq(self.entities['DATE'])
        entitiesFreq['Disaster'] = self.getEntitiesFreq(
            self.entities['Disaster'])
        filteredDates = []
        months = [
            'jan', 'feb', 'mar', 'apr', 'aug', 'sept', 'oct', 'nov', 'dec',
            'january', 'february', 'march', 'april', 'may', 'june', 'july',
            'august', 'september', 'october', 'november', 'december'
        ]
        for d, v in entitiesFreq['DATE']:
            if d.isdigit() and len(d) == 4:
                filteredDates.append((d, v))
            elif d.lower() in months:
                filteredDates.append((d, v))
        entitiesFreq['DATE'] = filteredDates

        llen = 5
        dlen = 5
        #l = [k for k,_ in entitiesFreq['LOCATION']]
        s = len(entitiesFreq['LOCATION'])

        if llen < s:
            s = llen
        t = entitiesFreq['LOCATION'][:s]
        print t
        self.entities['LOCATION'] = dict(t)

        #d = [k for k,_ in entitiesFreq['DATE']]
        s = len(entitiesFreq['DATE'])
        if dlen < s:
            s = dlen
        self.entities['DATE'] = dict(entitiesFreq['DATE'][:s])
        print entitiesFreq['DATE'][:s]

        locDate = [k for k, _ in entitiesFreq['LOCATION']
                   ] + [m for m, _ in entitiesFreq['DATE']]

        locDate = eventUtils.getTokens(' '.join(locDate))
        '''
        ntopToks = []
        topToks = [k for k,_ in sortedToksTFDF]
        for tok in topToks:
            if tok not in locDate:
                ntopToks.append(tok)
        topToks = ntopToks
        if self.topK < len(topToks):
            topToks =  topToks[:self.topK]
        '''

        ntopToks = []
        topToks = [k for k, _ in entitiesFreq['Disaster']]
        for tok in topToks:
            if tok not in locDate:
                ntopToks.append(tok)
        topToks = ntopToks
        if self.topK < len(topToks):
            topToks = topToks[:self.topK]
        #print "Disaster: ", topToks

        topToksDic = {}
        for t in topToks:
            topToksDic[t] = self.toksTFDFDic[t]
        #self.entities['Disaster'] = set(topToks)
        self.entities['Disaster'] = topToksDic
        #print self.entities
        print topToks

        #self.vecs = {}
        self.scalars = {}
        for k in self.entities:
            ekv = self.entities[k]
            '''
            if k == 'Disaster':
                ev = [1+math.log(e*v) for e,v in ekv.values()]
            else:
                ev = [1+math.log(e) for e in ekv.values()]
            '''
            #NoTFDF
            ev = [1 + math.log(e) for e in ekv.values()]
            #self.vecs[k] = ev
            self.scalars[k] = self.getScalar(ev)
예제 #19
0
def buildProbEventModel(docsList):
    t = ''
    docsTotalFreqs = []
    docsEntities = []
    docsEntitiesFreq = []
    entitiesProb = {}

    # Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
    # Also produces the total frequency for each document of each list (tokens, locations, and dates)
    for doc in docsList:

        if doc.has_key('text'):
            t = doc['text']
            if doc.has_key('title'):
                t = doc['title'] + " " + t
        if t:
            print 'Reading ' + t[:100]
            ents = eventUtils.getEntities(t)[0]
            docEnt = {}
            docEnt['LOCATION'] = {}
            if 'LOCATION' in ents:
                docEnt['LOCATION'] = ents['LOCATION']
            docEnt['DATE'] = {}
            if 'DATE' in ents:
                docEnt['DATE'] = ents['DATE']
            toks = eventUtils.getTokens(t)
            docEnt['Topic'] = toks
            docsEntities.append(docEnt)

            docEntFreq = {}
            #docTotals = {}
            for k in docEnt:
                docEntFreq[k] = eventUtils.getFreq(docEnt[k])
                #totalFreq = sum([v for _,v in docEntFreq[k].items()])

                #docTotals[k] = totalFreq
            docsEntitiesFreq.append(docEntFreq)
            #docsTotalFreqs.append(docTotals)

    # Collection-level frequency for each entity(tokens, locations, dates)

    # Total Frequency of keywords, locations, and dates in all documents
    '''
    allDocsTotal = {}
    allDocsTotal['LOCATION'] = 0
    allDocsTotal['DATE']=0
    allDocsTotal['Topic'] = 0
    
    for docTotFreq in docsTotalFreqs:
        for k in docTotFreq:
            allDocsTotal[k]+= docTotFreq[k]
    '''

    #Calculating prob for each item in each entity lists (tokens, locations, and dates) as
    # freq of item in all docs / total freq of all terms in that list
    entitiesProb['LOCATION'] = {}
    entitiesProb['DATE'] = {}
    entitiesProb['Topic'] = {}

    for docEntFreq in docsEntitiesFreq:
        for entity in docEntFreq:
            for val in docEntFreq[entity]:
                if val in entitiesProb[entity]:
                    entitiesProb[entity][val] += docEntFreq[entity][val]
                else:
                    entitiesProb[entity][val] = docEntFreq[entity][val]

    for ent in entitiesProb:
        allvalsFreq = sum([v for _, v in entitiesProb[ent].items()])
        for k in entitiesProb[ent]:
            #entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])

            entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] * 1.0)) / (
                len(entitiesProb[ent]) + allvalsFreq)

    return docsEntities, entitiesProb
예제 #20
0
    def buildEventModel(self, keywordsTh, seedURLs):

        corpus = Collection(seedURLs)

        # NoTFDF
        sortedToksTFDF = corpus.getIndicativeWords()
        self.toksTFDFDic = dict(sortedToksTFDF)
        # print sortedToksTFDF

        # sortedImptSents = corpus.getIndicativeSentences(self.topK,self.intersectionTh)
        sortedImptSents = corpus.getIndicativeSentences(keywordsTh, self.intersectionTh)
        # Get Event Model
        eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)

        self.entities["LOCATION"] = []
        self.entities["DATE"] = []
        self.entities["Disaster"] = []
        for e in eventModelInstances:
            if "LOCATION" in e:
                self.entities["LOCATION"].extend(e["LOCATION"])
            elif "DATE" in e:
                self.entities["DATE"].extend(e["DATE"])
            self.entities["Disaster"].extend(e["Disaster"])

        entitiesFreq = {}
        entitiesFreq["LOCATION"] = self.getEntitiesFreq(self.entities["LOCATION"])
        entitiesFreq["DATE"] = self.getEntitiesFreq(self.entities["DATE"])
        entitiesFreq["Disaster"] = self.getEntitiesFreq(self.entities["Disaster"])
        filteredDates = []
        months = [
            "jan",
            "feb",
            "mar",
            "apr",
            "aug",
            "sept",
            "oct",
            "nov",
            "dec",
            "january",
            "february",
            "march",
            "april",
            "may",
            "june",
            "july",
            "august",
            "september",
            "october",
            "november",
            "december",
        ]
        for d, v in entitiesFreq["DATE"]:
            if d.isdigit() and len(d) == 4:
                filteredDates.append((d, v))
            elif d.lower() in months:
                filteredDates.append((d, v))
        entitiesFreq["DATE"] = filteredDates

        llen = 5
        dlen = 5
        # l = [k for k,_ in entitiesFreq['LOCATION']]
        s = len(entitiesFreq["LOCATION"])

        if llen < s:
            s = llen
        t = entitiesFreq["LOCATION"][:s]
        print t
        self.entities["LOCATION"] = dict(t)

        # d = [k for k,_ in entitiesFreq['DATE']]
        s = len(entitiesFreq["DATE"])
        if dlen < s:
            s = dlen
        self.entities["DATE"] = dict(entitiesFreq["DATE"][:s])
        print entitiesFreq["DATE"][:s]

        locDate = [k for k, _ in entitiesFreq["LOCATION"]] + [m for m, _ in entitiesFreq["DATE"]]

        locDate = eventUtils.getTokens(" ".join(locDate))
        """
        ntopToks = []
        topToks = [k for k,_ in sortedToksTFDF]
        for tok in topToks:
            if tok not in locDate:
                ntopToks.append(tok)
        topToks = ntopToks
        if self.topK < len(topToks):
            topToks =  topToks[:self.topK]
        """

        ntopToks = []
        topToks = [k for k, _ in entitiesFreq["Disaster"]]
        for tok in topToks:
            if tok not in locDate:
                ntopToks.append(tok)
        topToks = ntopToks
        if self.topK < len(topToks):
            topToks = topToks[: self.topK]
        # print "Disaster: ", topToks

        topToksDic = {}
        for t in topToks:
            topToksDic[t] = self.toksTFDFDic[t]
        # self.entities['Disaster'] = set(topToks)
        self.entities["Disaster"] = topToksDic
        # print self.entities
        print topToks

        # self.vecs = {}
        self.scalars = {}
        for k in self.entities:
            ekv = self.entities[k]
            """
            if k == 'Disaster':
                ev = [1+math.log(e*v) for e,v in ekv.values()]
            else:
                ev = [1+math.log(e) for e in ekv.values()]
            """
            # NoTFDF
            ev = [1 + math.log(e) for e in ekv.values()]
            # self.vecs[k] = ev
            self.scalars[k] = self.getScalar(ev)
예제 #21
0
 def buildEventModel(self, seedURLs):
         
     corpus = Collection(seedURLs)
     
     #NoTFDF
     corpus.getIndicativeWords('TF')
     self.toksDic= dict(corpus.indicativeWords)
     #self.toksTFIDFDic = dict(sortedToksTFIDF)
     #print sortedToksTFDF
     
     sortedImptSents = corpus.getIndicativeSentences(3 * self.topK,self.intersectionTh)
     #sortedImptSents = corpus.getIndicativeSentences(keywordsTh,self.intersectionTh)
     for s in sortedImptSents[:self.topK]: 
         print s 
     # Get Event Model
     eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)
     #print eventModelInstances[:self.topK]
     
     self.entities['LOCATION']= []
     self.entities['DATE'] = []
     self.entities['Topic']=[]
     
     for e in eventModelInstances:
         if 'LOCATION' in e:
             self.entities['LOCATION'].extend( e['LOCATION'])
         if 'DATE' in e:
             self.entities['DATE'].extend( e['DATE'])
         self.entities['Topic'].extend(e['Topic'])
     
     entitiesFreq = {}
     entitiesFreq['LOCATION'] = self.getEntitiesFreq(self.entities['LOCATION'])
     entitiesFreq['DATE'] = self.getEntitiesFreq(self.entities['DATE'])
     entitiesFreq['Topic'] = self.getEntitiesFreq(self.entities['Topic'])
     #entitiesFreq['Topic'] = [(t,self.toksTFIDFDic[t]) for t,f in tf ]
     '''
     if self.topK < len(entitiesFreq['Topic']):
         entitiesFreq['Topic'] = entitiesFreq['Topic'][:self.topK]
     self.entities['Topic'] = dict(entitiesFreq['Topic'])
     print entitiesFreq['Topic']
     '''
     filteredDates = []
     months = ['jan','feb','mar','apr','aug','sept','oct','nov','dec','january','february','march','april','may','june','july','august','september','october','november','december']
     for d,v in entitiesFreq['DATE']:
         if d.isdigit() and len(d) == 4:
             filteredDates.append((d,v))
         elif d.lower() in months:
             filteredDates.append((d,v))
     entitiesFreq['DATE']=filteredDates
     
     llen = self.topK
     dlen = self.topK
     #l = [k for k,_ in entitiesFreq['LOCATION']]
     s = len(entitiesFreq['LOCATION'])
     
     if llen < s:
         s = llen
     t = entitiesFreq['LOCATION'][:s]
     print t
     self.entities['LOCATION'] = dict(t)
            
     #locDate = [k for k,_ in entitiesFreq['LOCATION']] + [m for m,_ in entitiesFreq['DATE']]
     locDate = self.entities['LOCATION'].keys() + [m for m,_ in entitiesFreq['DATE']]#self.entities['DATE'].keys()
     
     locDate = eventUtils.getTokens(' '.join(locDate))
     
     #d = [k for k,_ in entitiesFreq['DATE']]
     s = len(entitiesFreq['DATE'])
     if dlen < s:
         s = dlen
     self.entities['DATE'] = dict(entitiesFreq['DATE'][:s])
     print entitiesFreq['DATE'][:s]
     
     
     ntopToks = []
     topToks = [k for k,_ in entitiesFreq['Topic']]
     for tok in topToks:
         if tok not in locDate:
             ntopToks.append(tok)
     topToks = ntopToks
     
     if self.topK < len(topToks):
         topToks =  topToks[:self.topK]
     #print "Disaster: ", topToks
     
     
     topToksDic = {}
     for t in topToks:
         topToksDic[t] = self.toksDic[t]
     #self.entities['Disaster'] = set(topToks)
     self.entities['Topic'] = topToksDic
     
     #print self.entities
     print topToksDic
     
     #Calculate weights
     self.calculateWeights()
     
     newents = {}
     for k in self.entities:
         ed = self.entities[k].iteritems()
         ned = [(k,1) for k,_ in ed]
         newents[k] = dict(ned)
     
     for k in newents:
         self.entities[k] = newents[k]
             
         
     
     #self.vecs = {}
     self.scalars = {}
     for k in self.entities:
         ekv = self.entities[k]
         '''
         if k == 'Disaster':
             ev = [1+math.log(e*v) for e,v in ekv.values()]
         else:
             ev = [1+math.log(e) for e in ekv.values()]
         '''
         #NoTFDF
         ev = [1+math.log(e) for e in ekv.values()]
         #self.vecs[k] = ev
         self.scalars[k] = self.getScalar(ev)
예제 #22
0
 def calculate_similarity(self,doc):
     #weigths ={'Topic':0.0,'LOCATION':0.0, 'DATE':0.0}
     '''
     entFreq = {}
     for k in self.entities:
         entFreq[k]= sum(self.entities[k].values())
     totFreq = sum(entFreq.values())
     
     for k in weigths:
         weigths[k] = entFreq[k]*1.0 / totFreq
     '''
     topicDic = self.entities['Topic']
     
     locToks = self.entities['LOCATION'].keys()
     locToks = eventUtils.getStemmedWords(locToks)
     locDic = dict(zip(locToks,self.entities['LOCATION'].values()))
     
     dToks = self.entities['DATE'].keys()
     dToks = eventUtils.getStemmedWords(dToks)
     dDic = dict(zip(dToks,self.entities['DATE'].values()))
     
     tokens = eventUtils.getTokens(doc)
     tokensDic = eventUtils.getFreq(tokens)
     wv = [1+math.log(e) for e in tokensDic.values()]
     wvScalar = self.getScalar(wv)
     scores = []
     
     ksd = 0    
     #interst = 0
     for i in tokensDic:
         if i in topicDic:
             ksd += (1+math.log(topicDic[i]))* (1+math.log(tokensDic[i]))
             #interst +=1
     if ksd != 0:
         ksd = float(ksd)/(self.scalars['Topic'] * wvScalar)
     #else:
     #    ksd = 0
     #if ksd == 0:
     #    return 0
     #if interst < 2:
         #return 0
     scores.append(ksd*self.weights['Topic'])
     ksl = 0    
     for i in tokensDic:
         if i in locDic:
             ksl += (1+math.log(locDic[i]))* (1+math.log(tokensDic[i]))
     if ksl != 0:
         ksl = float(ksl)/(self.scalars['LOCATION'] * wvScalar) 
     #else:
     #    ksl = 0
     scores.append(ksl*self.weights['LOCATION'])
     
     ks = 0    
     for i in tokensDic:
         if i in dDic:
             ks += (1+math.log(dDic[i]))* (1+math.log(tokensDic[i]))
     if ks != 0:
         ks = float(ks)/(self.scalars['DATE'] * wvScalar)    
     #else:
     #    ks = 0
     scores.append(ks*self.weights['DATE'])
     
     #score = sum(scores) / 3.0
     score = sum(scores)
     return score
예제 #23
0
 def buildEventModel_old(self,seedURLs):
     
     corpus = Collection(seedURLs)
     #sortedTokensFreqs = corpus.getWordsFrequencies()
     sortedToksTFDF = corpus.getIndicativeWords()
     print sortedToksTFDF
     sortedImptSents = corpus.getIndicativeSentences(self.topK,self.intersectionTh)
     # Get Event Model
     eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)
     #topToks = [k for k,_ in sortedToksTFDF]
     #if self.topK < len(topToks):
     #    topToks =  topToks[:self.topK]
     #self.entities['Disaster'] = set(topToks)
     
     self.entities['LOCATION']= []
     self.entities['DATE'] = []
     for e in eventModelInstances:
         if 'LOCATION' in e:
             self.entities['LOCATION'].extend( e['LOCATION'])
         elif 'DATE' in e:
             self.entities['DATE'].extend( e['DATE'])
     
     entitiesFreq = {}
     entitiesFreq['LOCATION'] = eventUtils.getFreq(self.entities['LOCATION'])
     entitiesFreq['LOCATION'] = eventUtils.getSorted(entitiesFreq['LOCATION'].items(), 1)
     entitiesFreq['DATE'] = eventUtils.getFreq(self.entities['DATE'])
     entitiesFreq['DATE'] = eventUtils.getSorted(entitiesFreq['DATE'].items(), 1)
     
     l = [k for k,_ in entitiesFreq['LOCATION']]
     if self.topK < len(l):
         #l = l[:self.topK]
         l = l[:3]
     self.entities['LOCATION'] = set(l)
     
     d = [k for k,_ in entitiesFreq['DATE']]
     if self.topK < len(d):
         #d = d[:self.topK]
         d = d[:3]
     self.entities['DATE'] = set(d)
     
     self.entities['LOCATION'] = self.getUniqueEntities(self.entities['LOCATION'])
     
     
     self.entities['DATE'] = self.getUniqueEntities(self.entities['DATE']) 
     
     locDate = list(self.entities['LOCATION']) + list(self.entities['DATE'])
     locDate = eventUtils.getTokens(' '.join(locDate))
     
     ntopToks = []
     topToks = [k for k,_ in sortedToksTFDF]
     for tok in topToks:
         if tok not in locDate:
             ntopToks.append(tok)
     topToks = ntopToks
     if self.topK < len(topToks):
         topToks =  topToks[:self.topK]
     self.entities['Disaster'] = set(topToks)
     
     
     self.allEntities = []
     for k in self.entities:
         self.allEntities.extend(self.entities[k]) 
         
     print self.allEntities
예제 #24
0
 def buildEventModel(self,keywordsTh, seedURLs):
     
     corpus = Collection(seedURLs)
     
     #NoTFDF
     sortedToksTFDF = corpus.getIndicativeWords()
     self.toksTFDFDic = dict(sortedToksTFDF)
     print sortedToksTFDF
     
     #sortedImptSents = corpus.getIndicativeSentences(self.topK,self.intersectionTh)
     sortedImptSents = corpus.getIndicativeSentences(keywordsTh,self.intersectionTh)
     # Get Event Model
     eventModelInstances = eventUtils.getEventModelInsts(sortedImptSents)
     
     
     self.entities['LOCATION']= []
     self.entities['DATE'] = []
     self.entities['Disaster']=[]
     for e in eventModelInstances:
         if 'LOCATION' in e:
             self.entities['LOCATION'].extend( e['LOCATION'])
         elif 'DATE' in e:
             self.entities['DATE'].extend( e['DATE'])
         self.entities['Disaster'].extend(e['Disaster'])
     
     entitiesFreq = {}
     entitiesFreq['LOCATION'] = self.getEntitiesFreq(self.entities['LOCATION'])
     entitiesFreq['DATE'] = self.getEntitiesFreq(self.entities['DATE'])
     entitiesFreq['Disaster'] = self.getEntitiesFreq(self.entities['Disaster'])
     filteredDates = []
     months = ['january','february','march','april','may','june','july','august','september','october','november','december']
     for d,v in entitiesFreq['DATE']:
         if d.isdigit() and len(d) == 4:
             filteredDates.append((d,v))
         elif d.lower() in months:
             filteredDates.append((d,v))
     entitiesFreq['DATE']=filteredDates
     
     llen = 5
     dlen = 5
     #l = [k for k,_ in entitiesFreq['LOCATION']]
     s = len(entitiesFreq['LOCATION'])
     
     if llen < s:
         s = llen
     t = entitiesFreq['LOCATION'][:s]
     print t
     self.entities['LOCATION'] = dict(t)
            
     #d = [k for k,_ in entitiesFreq['DATE']]
     s = len(entitiesFreq['DATE'])
     if dlen < s:
         s = dlen
     self.entities['DATE'] = dict(entitiesFreq['DATE'][:s])
     print entitiesFreq['DATE'][:s]
     
     
     locDate = [k for k,_ in entitiesFreq['LOCATION'][:2]] + [m for m,_ in entitiesFreq['DATE']]
     
     locDate = eventUtils.getTokens(' '.join(locDate))
     
     ntopToks = []
     topToks = [k for k,_ in entitiesFreq['Disaster']]
     for tok in topToks:
         if tok not in locDate:
             ntopToks.append(tok)
     topToks = ntopToks
     if self.topK < len(topToks):
         topToks =  topToks[:self.topK]
     #print "Disaster: ", topToks
     
     
     topToksDic = {}
     for t in topToks:
         topToksDic[t] = self.toksTFDFDic[t]
     #self.entities['Disaster'] = set(topToks)
     self.entities['Disaster'] = topToksDic
     #print self.entities
     print topToks
     
     #self.vecs = {}
     self.scalars = {}
     for k in self.entities:
         ekv = self.entities[k]
         
         #NoTFDF
         ev = [1+math.log(e) for e in ekv.values()]
         #self.vecs[k] = ev
         self.scalars[k] = self.getScalar(ev)
예제 #25
0
 def buildProbEventModel(self,urlsList,topK):
     
     docsList = eventUtils.getWebpageText(urlsList) #self.getCollectionDocs(urlsList)
     t = ''
     #docsTotalFreqs=[]
     docsEntities=[]
     docsEntitiesFreq = []
     entitiesProb = {}
     
     # Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
     # Also produces the total frequency for each document of each list (tokens, locations, and dates)
     for doc in docsList:
         
         if doc.has_key('text'):
             t = doc['text']
             if doc.has_key('title'):
                 t =doc['title']+ " "+t
         if t:
             #print 'Reading ' + t[:100]
             ents = eventUtils.getEntities(t)[0]
             docEnt = {}
             docEnt['LOCATION']={}
             if 'LOCATION' in ents:
                 docEnt['LOCATION'] =  ents['LOCATION']
             docEnt['DATE']={}
             if 'DATE' in ents:
                 docEnt['DATE'] = ents['DATE']
             toks = eventUtils.getTokens(t)
             docEnt['Topic'] = toks
             docsEntities.append(docEnt)
             
             docEntFreq = {}
             #docTotals = {}
             for k in docEnt:
                 docEntFreq[k] = eventUtils.getFreq(docEnt[k])
                 #totalFreq = sum([v for _,v in docEntFreq[k].items()])
                 
                 #docTotals[k] = totalFreq
             docsEntitiesFreq.append(docEntFreq)
             #docsTotalFreqs.append(docTotals)
     
     # Collection-level frequency for each entity(tokens, locations, dates)
     
     #Calculating prob for each item in each entity lists (tokens, locations, and dates) as 
     # freq of item in all docs / total freq of all terms in that list
     entitiesProb['LOCATION']={}
     entitiesProb['DATE']={}
     entitiesProb['Topic']={}
     
     for docEntFreq in docsEntitiesFreq:
         for entity in docEntFreq:
             for val in docEntFreq[entity]:
                 if val in entitiesProb[entity]:
                     entitiesProb[entity][val] += docEntFreq[entity][val]
                 else:
                     entitiesProb[entity][val] = docEntFreq[entity][val]
     
     for ent in entitiesProb:
         allvalsFreq = sum([v for _,v in entitiesProb[ent].items()])
         for k in entitiesProb[ent]:
             #entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])
             
             entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (len(entitiesProb[ent]) + allvalsFreq)
             
         
     #self.probEvtModel = entitiesProb
     
     mle =  self.getMLEEventEntities(entitiesProb,10)
     for k in mle:
         print k, mle[k]
         
     
     self.probEvtModel = {}
     for k in mle:
         self.probEvtModel[k] = dict(mle[k])#entitiesProb[k][:topK]
     
     self.eDisDic = self.probEvtModel['Topic']
     
     
     locToks = self.probEvtModel['LOCATION'].keys()
     locToks = eventUtils.getStemmedWords(locToks)
     self.locDic = dict(zip(locToks,self.probEvtModel['LOCATION'].values()))
     
 
     dToks = self.probEvtModel['DATE'].keys()
     dToks = eventUtils.getStemmedWords(dToks)
     self.dDic = dict(zip(dToks,self.probEvtModel['DATE'].values()))
     
     
     
     return docsEntities, entitiesProb
예제 #26
0
    def buildProbEventModel(self, urlsList, topK):

        docsList = eventUtils.getWebpageText_NoURLs(urlsList)  #getWebpageText
        docsList = [d for d in docsList if 'text' in d]
        t = ''
        #docsTotalFreqs=[]
        docsEntities = []
        docsEntitiesFreq = []
        entitiesFreq = {}

        # Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
        # Also produces the total frequency for each document of each list (tokens, locations, and dates)
        for doc in docsList:
            #t = ""
            #if doc.has_key('text'):
            t = doc['text']
            #if doc.has_key('title'):
            #    t =doc['title']+ " "+t
            #if t:
            #print 'Reading ' + t[:100]
            ents = eventUtils.getEntities(t)[0]
            docEnt = {}
            docEnt['LOCATION'] = {}
            if 'LOCATION' in ents:
                docEnt['LOCATION'] = ents['LOCATION']
            docEnt['DATE'] = {}
            if 'DATE' in ents:
                docEnt['DATE'] = ents['DATE']
            toks = eventUtils.getTokens(t)
            docEnt['Topic'] = toks
            docsEntities.append(docEnt)

            docEntFreq = {}
            #docTotals = {}
            for k in docEnt:
                docEntFreq[k] = eventUtils.getFreq(docEnt[k])
                #totalFreq = sum([v for _,v in docEntFreq[k].items()])

                #docTotals[k] = totalFreq
            docsEntitiesFreq.append(docEntFreq)
            #docsTotalFreqs.append(docTotals)

        # Collection-level frequency for each entity(tokens, locations, dates)

        #Calculating prob for each item in each entity lists (tokens, locations, and dates) as
        # freq of item in all docs / total freq of all terms in that list
        entitiesFreq['LOCATION'] = defaultdict(float)  #{}
        entitiesFreq['DATE'] = defaultdict(float)  #{}
        entitiesFreq['Topic'] = defaultdict(float)  #{}

        for docEntFreq in docsEntitiesFreq:
            for entity in docEntFreq:
                for val in docEntFreq[entity]:
                    #if val in entitiesProb[entity]:
                    entitiesFreq[entity][val] += docEntFreq[entity][val]
                    #else:
                    #    entitiesProb[entity][val] = docEntFreq[entity][val]
        self.defaultProb = {}
        entitiesProb = {}
        for ent in entitiesFreq:
            allvalsFreq = sum([v for _, v in entitiesFreq[ent].items()])
            l = len(entitiesFreq[ent])
            denom = l + allvalsFreq
            self.defaultProb[ent] = 1.0 / denom
            entitiesProb[ent] = defaultdict(lambda: 1.0 / denom)
            for k in entitiesFreq[ent]:
                #entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])

                entitiesProb[ent][k] = (
                    1.0 + entitiesProb[ent][k]) / denom  #(l + allvalsFreq)

        #self.probEvtModel = entitiesProb

        mle = self.getMLEEventEntities(entitiesProb, 10)
        for k in mle:
            print k, mle[k]

        self.probEvtModel = {}
        for k in mle:
            #self.probEvtModel[k] = dict(mle[k])#entitiesProb[k][:topK]
            self.probEvtModel[k] = defaultdict(lambda: self.defaultProb[k])
            for e, v in mle[k]:
                self.probEvtModel[k][e] = v

        #self.eDisDic = self.probEvtModel['Topic']

        locToks = self.probEvtModel['LOCATION'].keys()
        locToks = eventUtils.getStemmedWords(locToks)
        #self.locDic = dict(zip(locToks,self.probEvtModel['LOCATION'].values()))
        locDic = defaultdict(lambda: self.defaultProb['LOCATION'])
        for k, v in zip(locToks, self.probEvtModel['LOCATION'].values()):
            locDic[k] = v
        self.probEvtModel['LOCATION'] = locDic

        dToks = self.probEvtModel['DATE'].keys()
        dToks = eventUtils.getStemmedWords(dToks)
        #self.dDic = dict(zip(dToks,self.probEvtModel['DATE'].values()))
        dDic = defaultdict(lambda: self.defaultProb['DATE'])
        for k, v in zip(locToks, self.probEvtModel['DATE'].values()):
            dDic[k] = v
        self.probEvtModel['DATE'] = dDic

        return docsEntities, entitiesProb