site.addsitedir('/bos/usr0/cx/PyCode/cxPyLib')
site.addsitedir('/bos/usr0/cx/PyCode/GoogleAPI')
from ObjCenter.FbObjCacheCenter import FbObjCacheCenterC
from cxBase.Conf import cxConfC
import sys
if 2 != len(sys.argv):
FbObjCacheCenterC.ShowConf()
print "in\nout"
sys.exit()
ObjCenter = FbObjCacheCenterC(sys.argv[1])
conf = cxConfC(sys.argv[1])
InName = conf.GetConf('in')
OutName = conf.GetConf('out')
out = open(OutName, 'w')
cnt = 0
for line in open(InName):
line = line.strip()
vCol = line.split('\t')
lName = [ObjCenter.FetchObjName(ObjId) for ObjId in vCol[:2]]
print >> out, line + '\t' + '\t'.join(lName)
cnt += 1
if 0 == (cnt % 100):
print "processed [%d] line" % (cnt)
out.close()
class EntityCorrelationFromTextSimC(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.NeighborNum = 50
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.Searcher.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
self.NeighborNum = self.conf.GetConf('neighbornum', self.NeighborNum)
@staticmethod
def ShowConf():
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'neighbornum'
def ProcessOneObj(self, ObjId, name):
'''
return lObjNeighbor=[objid,KL score] top self.NeighborNum
'''
#search in index, get top 1000
query = TextBaseC.RawClean(name)
if "" == query:
return []
lObjDoc = self.Searcher.RunQuery(query)
lObjNeighbor = []
ThisDesp = self.ObjCenter.FetchObjDesp(ObjId)
ThisLm = LmBaseC(ThisDesp)
ThisVec = VectorC(ThisLm.hTermTF)
# print '[%s:%s] desp : [%s]' %(ObjId,name,ThisDesp)
if len(ThisLm.hTermTF) == 0:
return []
for ObjDoc in lObjDoc:
Id = ObjDoc.DocNo
if Id == ObjId:
continue
if not Id.startswith('/m/'):
print "[%s %s] neighbor id [%s] format error" % (ObjId, name,
Id)
continue
# print "get neighbor [%s] [%s]" %(Id,ObjDoc.GetContent())
# NeighborDesp = ObjDoc.GetContent()
NeighborLm = LmBaseC(ObjDoc)
NeighborVec = VectorC(NeighborLm.hTermTF)
if len(NeighborVec.hDim) == 0:
continue
score = VectorC.KL(ThisVec, NeighborVec)
lObjNeighbor.append([Id, -score])
# print "[%s %s] KL [%f]" %(ObjId,Id,score)
# print "%s\n%s" %(json.dumps(ThisVec.hDim),json.dumps(NeighborVec.hDim))
lObjNeighbor.sort(key=lambda item: item[1], reverse=True)
print "[%s:%s] neighbor id score get" % (ObjId, name)
return lObjNeighbor
def Process(self, ObjInName, OutName):
out = open(OutName, 'w')
for line in open(ObjInName):
vCol = line.strip().split('\t')
if len(vCol) < 2:
continue
lObjNeighbor = self.ProcessOneObj(vCol[0], vCol[1])
for NeighborId, score in lObjNeighbor[:self.NeighborNum]:
print >> out, '%s\t%s\t%f\t%s\t%s' % (
vCol[0], NeighborId, score, vCol[1],
self.ObjCenter.FetchObjName(NeighborId))
print "[%s:%s] done" % (vCol[0], vCol[1])
out.close()
print "finished"