def calculate_similarity_equalWeights_duplicate(self,doc):
eDisDic = self.entities['Topic']
locToks = self.entities['LOCATION'].keys()
locToks = eventUtils.getStemmedWords(locToks)
locDic = dict(zip(locToks,self.entities['LOCATION'].values()))
dToks = self.entities['DATE'].keys()
dToks = eventUtils.getStemmedWords(dToks)
dDic = dict(zip(dToks,self.entities['DATE'].values()))
tokens = eventUtils.getTokens(doc)
tokensDic = eventUtils.getFreq(tokens)
wv = [1+math.log(e) for e in tokensDic.values()]
wvScalar = self.getScalar(wv)
scores = []
ksd = 0
#interst = 0
for i in tokensDic:
if i in eDisDic:
ksd += (1+math.log(eDisDic[i]))* (1+math.log(tokensDic[i]))
#interst +=1
if ksd > 0:
ksd = float(ksd)/(self.scalars['Topic'] * wvScalar)
else:
ksd = 0
if ksd == 0:
return 0
#if interst < 2:
#return 0
scores.append(ksd)
ksl = 0
for i in tokensDic:
if i in locDic:
ksl += (1+math.log(locDic[i]))* (1+math.log(tokensDic[i]))
if ksl > 0:
ksl = float(ksl)/(self.scalars['LOCATION'] * wvScalar)
else:
ksl = 0
scores.append(ksl)
ks = 0
for i in tokensDic:
if i in dDic:
ks += (1+math.log(dDic[i]))* (1+math.log(tokensDic[i]))
if ks > 0:
ks = float(ks)/(self.scalars['DATE'] * wvScalar)
else:
ks = 0
scores.append(ks)
score = sum(scores) / 3.0
return score
def calculate_similarity(self, doc):
eDisDic = self.entities["Disaster"]
locToks = self.entities["LOCATION"].keys()
locToks = eventUtils.getStemmedWords(locToks)
locDic = dict(zip(locToks, self.entities["LOCATION"].values()))
dToks = self.entities["DATE"].keys()
dToks = eventUtils.getStemmedWords(dToks)
dDic = dict(zip(dToks, self.entities["DATE"].values()))
tokens = eventUtils.getTokens(doc)
tokensDic = eventUtils.getFreq(tokens)
wv = [1 + math.log(e) for e in tokensDic.values()]
wvScalar = self.getScalar(wv)
scores = []
ksd = 0
for i in tokensDic:
if i in eDisDic:
ksd += (1 + math.log(eDisDic[i])) * (1 + math.log(tokensDic[i]))
if ksd > 0:
ksd = float(ksd) / (self.scalars["Disaster"] * wvScalar)
else:
ksd = 0
if ksd == 0:
return 0
scores.append(ksd)
ksl = 0
for i in tokensDic:
if i in locDic:
ksl += (1 + math.log(locDic[i])) * (1 + math.log(tokensDic[i]))
if ksl > 0:
ksl = float(ksl) / (self.scalars["LOCATION"] * wvScalar)
else:
ksl = 0
scores.append(ksl)
ks = 0
for i in tokensDic:
if i in dDic:
ks += (1 + math.log(dDic[i])) * (1 + math.log(tokensDic[i]))
if ks > 0:
ks = float(ks) / (self.scalars["DATE"] * wvScalar)
else:
ks = 0
scores.append(ks)
score = sum(scores) / 3.0
return score
def buildProbEventModel(self,urlsList,topK):
docsList = eventUtils.getWebpageText(urlsList) #self.getCollectionDocs(urlsList)
t = ''
#docsTotalFreqs=[]
docsEntities=[]
docsEntitiesFreq = []
entitiesProb = {}
# Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
# Also produces the total frequency for each document of each list (tokens, locations, and dates)
for doc in docsList:
if doc.has_key('text'):
t = doc['text']
if doc.has_key('title'):
t =doc['title']+ " "+t
if t:
#print 'Reading ' + t[:100]
ents = eventUtils.getEntities(t)[0]
docEnt = {}
docEnt['LOCATION']={}
if 'LOCATION' in ents:
docEnt['LOCATION'] = ents['LOCATION']
docEnt['DATE']={}
if 'DATE' in ents:
docEnt['DATE'] = ents['DATE']
toks = eventUtils.getTokens(t)
docEnt['Topic'] = toks
docsEntities.append(docEnt)
docEntFreq = {}
#docTotals = {}
for k in docEnt:
docEntFreq[k] = eventUtils.getFreq(docEnt[k])
#totalFreq = sum([v for _,v in docEntFreq[k].items()])
#docTotals[k] = totalFreq
docsEntitiesFreq.append(docEntFreq)
#docsTotalFreqs.append(docTotals)
# Collection-level frequency for each entity(tokens, locations, dates)
#Calculating prob for each item in each entity lists (tokens, locations, and dates) as
# freq of item in all docs / total freq of all terms in that list
entitiesProb['LOCATION']={}
entitiesProb['DATE']={}
entitiesProb['Topic']={}
for docEntFreq in docsEntitiesFreq:
for entity in docEntFreq:
for val in docEntFreq[entity]:
if val in entitiesProb[entity]:
entitiesProb[entity][val] += docEntFreq[entity][val]
else:
entitiesProb[entity][val] = docEntFreq[entity][val]
for ent in entitiesProb:
allvalsFreq = sum([v for _,v in entitiesProb[ent].items()])
for k in entitiesProb[ent]:
#entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])
entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (len(entitiesProb[ent]) + allvalsFreq)
#self.probEvtModel = entitiesProb
mle = self.getMLEEventEntities(entitiesProb,10)
for k in mle:
print k, mle[k]
self.probEvtModel = {}
for k in mle:
self.probEvtModel[k] = dict(mle[k])#entitiesProb[k][:topK]
self.eDisDic = self.probEvtModel['Topic']
locToks = self.probEvtModel['LOCATION'].keys()
locToks = eventUtils.getStemmedWords(locToks)
self.locDic = dict(zip(locToks,self.probEvtModel['LOCATION'].values()))
dToks = self.probEvtModel['DATE'].keys()
dToks = eventUtils.getStemmedWords(dToks)
self.dDic = dict(zip(dToks,self.probEvtModel['DATE'].values()))
return docsEntities, entitiesProb
def calculate_similarity(self,doc):
#weigths ={'Topic':0.0,'LOCATION':0.0, 'DATE':0.0}
'''
entFreq = {}
for k in self.entities:
entFreq[k]= sum(self.entities[k].values())
totFreq = sum(entFreq.values())
for k in weigths:
weigths[k] = entFreq[k]*1.0 / totFreq
'''
topicDic = self.entities['Topic']
locToks = self.entities['LOCATION'].keys()
locToks = eventUtils.getStemmedWords(locToks)
locDic = dict(zip(locToks,self.entities['LOCATION'].values()))
dToks = self.entities['DATE'].keys()
dToks = eventUtils.getStemmedWords(dToks)
dDic = dict(zip(dToks,self.entities['DATE'].values()))
tokens = eventUtils.getTokens(doc)
tokensDic = eventUtils.getFreq(tokens)
wv = [1+math.log(e) for e in tokensDic.values()]
wvScalar = self.getScalar(wv)
scores = []
ksd = 0
#interst = 0
for i in tokensDic:
if i in topicDic:
ksd += (1+math.log(topicDic[i]))* (1+math.log(tokensDic[i]))
#interst +=1
if ksd != 0:
ksd = float(ksd)/(self.scalars['Topic'] * wvScalar)
#else:
# ksd = 0
#if ksd == 0:
# return 0
#if interst < 2:
#return 0
scores.append(ksd*self.weights['Topic'])
ksl = 0
for i in tokensDic:
if i in locDic:
ksl += (1+math.log(locDic[i]))* (1+math.log(tokensDic[i]))
if ksl != 0:
ksl = float(ksl)/(self.scalars['LOCATION'] * wvScalar)
#else:
# ksl = 0
scores.append(ksl*self.weights['LOCATION'])
ks = 0
for i in tokensDic:
if i in dDic:
ks += (1+math.log(dDic[i]))* (1+math.log(tokensDic[i]))
if ks != 0:
ks = float(ks)/(self.scalars['DATE'] * wvScalar)
#else:
# ks = 0
scores.append(ks*self.weights['DATE'])
#score = sum(scores) / 3.0
score = sum(scores)
return score
def buildProbEventModel(self, urlsList, topK):
docsList = eventUtils.getWebpageText_NoURLs(urlsList) #getWebpageText
docsList = [d for d in docsList if 'text' in d]
t = ''
#docsTotalFreqs=[]
docsEntities = []
docsEntitiesFreq = []
entitiesFreq = {}
# Convert each doc to tokens, locations, dates lists and their corresponding frequency distributions
# Also produces the total frequency for each document of each list (tokens, locations, and dates)
for doc in docsList:
#t = ""
#if doc.has_key('text'):
t = doc['text']
#if doc.has_key('title'):
# t =doc['title']+ " "+t
#if t:
#print 'Reading ' + t[:100]
ents = eventUtils.getEntities(t)[0]
docEnt = {}
docEnt['LOCATION'] = {}
if 'LOCATION' in ents:
docEnt['LOCATION'] = ents['LOCATION']
docEnt['DATE'] = {}
if 'DATE' in ents:
docEnt['DATE'] = ents['DATE']
toks = eventUtils.getTokens(t)
docEnt['Topic'] = toks
docsEntities.append(docEnt)
docEntFreq = {}
#docTotals = {}
for k in docEnt:
docEntFreq[k] = eventUtils.getFreq(docEnt[k])
#totalFreq = sum([v for _,v in docEntFreq[k].items()])
#docTotals[k] = totalFreq
docsEntitiesFreq.append(docEntFreq)
#docsTotalFreqs.append(docTotals)
# Collection-level frequency for each entity(tokens, locations, dates)
#Calculating prob for each item in each entity lists (tokens, locations, and dates) as
# freq of item in all docs / total freq of all terms in that list
entitiesFreq['LOCATION'] = defaultdict(float) #{}
entitiesFreq['DATE'] = defaultdict(float) #{}
entitiesFreq['Topic'] = defaultdict(float) #{}
for docEntFreq in docsEntitiesFreq:
for entity in docEntFreq:
for val in docEntFreq[entity]:
#if val in entitiesProb[entity]:
entitiesFreq[entity][val] += docEntFreq[entity][val]
#else:
# entitiesProb[entity][val] = docEntFreq[entity][val]
self.defaultProb = {}
entitiesProb = {}
for ent in entitiesFreq:
allvalsFreq = sum([v for _, v in entitiesFreq[ent].items()])
l = len(entitiesFreq[ent])
denom = l + allvalsFreq
self.defaultProb[ent] = 1.0 / denom
entitiesProb[ent] = defaultdict(lambda: 1.0 / denom)
for k in entitiesFreq[ent]:
#entitiesProb[ent][k] = (1.0 + (entitiesProb[ent][k] *1.0)) / (docsTotalFreqs[ent] + allDocsTotal[ent])
entitiesProb[ent][k] = (
1.0 + entitiesProb[ent][k]) / denom #(l + allvalsFreq)
#self.probEvtModel = entitiesProb
mle = self.getMLEEventEntities(entitiesProb, 10)
for k in mle:
print k, mle[k]
self.probEvtModel = {}
for k in mle:
#self.probEvtModel[k] = dict(mle[k])#entitiesProb[k][:topK]
self.probEvtModel[k] = defaultdict(lambda: self.defaultProb[k])
for e, v in mle[k]:
self.probEvtModel[k][e] = v
#self.eDisDic = self.probEvtModel['Topic']
locToks = self.probEvtModel['LOCATION'].keys()
locToks = eventUtils.getStemmedWords(locToks)
#self.locDic = dict(zip(locToks,self.probEvtModel['LOCATION'].values()))
locDic = defaultdict(lambda: self.defaultProb['LOCATION'])
for k, v in zip(locToks, self.probEvtModel['LOCATION'].values()):
locDic[k] = v
self.probEvtModel['LOCATION'] = locDic
dToks = self.probEvtModel['DATE'].keys()
dToks = eventUtils.getStemmedWords(dToks)
#self.dDic = dict(zip(dToks,self.probEvtModel['DATE'].values()))
dDic = defaultdict(lambda: self.defaultProb['DATE'])
for k, v in zip(locToks, self.probEvtModel['DATE'].values()):
dDic[k] = v
self.probEvtModel['DATE'] = dDic
return docsEntities, entitiesProb