def Init(self):
BoeLmC.Init(self)
self.DocTextDir = ""
self.ObjCenter = FbObjCacheCenterC()
self.CtfCenter = TermCtfC()
self.lInferenceWeight = [1,0,0]
self.hDocText = {}
def Init(self):
cxBaseC.Init(self)
self.ObjCenter = FbObjCacheCenterC()
self.Inferener = LESInferencerC()
self.DocKgDir = ""
self.hQObj = {}
self.OrigQWeight = 0.5
def Init(self):
cxBaseC.Init(self)
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.Evaluator = AdhocEvaC()
self.Inferener = LESInferencerC()
self.QDocNodeDataDir = ""
self.OrigQWeight = 0.5
self.UseQObjOnly = True
def Init(self):
cxBaseC.Init(self)
self.NodeDir = ""
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.QDocFeatureExtractor = LeToRFeatureExtractCenterC()
self.QObjFeatureExtractor = FbQObjFeatureExtractCenterC()
self.DocObjFeatureExtractor = FbObjDocFeatureExtractCenterC()
self.ObjObjFeatureExtractor = ObjObjFeatureExtractCenterC()
def Init(self):
cxBaseC.Init(self)
self.lQObjFeatureGroup = []
self.lObjObjFeatureGroup = []
self.lDocObjFeatureGroup = []
self.QObjAnaExtractor = QueryObjEdgeFeatureAnaExtractorC()
self.DocObjFaccExtractor = DocObjEdgeFeatureFaccExtractorC()
self.ObjObjKGExtractor = ObjObjEdgeFeatureKGExtractorC()
self.ObjObjPreCalcExtractor = ObjObjEdgeFeaturePreCalcSimExtractorC()
self.ObjObjTextSimExtractor = ObjObjEdgeFeatureTextSimExtractorC()
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.NodeDir = ""
def ShowConf(cls):
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
AdhocEvaC.ShowConf()
print 'qdocnodedatadir\norigqweight 0.5\nqobjonly 1'
class RawGraphPerEdgeFeatureConstructorC(SearchResDocGraphConstructorC):
def Init(self):
SearchResDocGraphConstructorC.Init(self)
self.EdgeFeatureCenter = ObjObjFeatureExtractCenterC()
self.ObjCenter = FbObjCacheCenterC()
def SetConf(self, ConfIn):
SearchResDocGraphConstructorC.SetConf(self, ConfIn)
self.EdgeFeatureCenter.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
@staticmethod
def ShowConf():
SearchResDocGraphConstructorC.ShowConf()
ObjObjFeatureExtractCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
def FormForOneQ(self, qid, query):
lDoc = self.Searcher.RunQuery(query, qid)
lDocKg = [self.GraphFormer.FillDocGraph(doc.DocNo) for doc in lDoc]
for DocKg in lDocKg:
logging.info('forming edge mtx for [%s] [%d] obj', DocKg.DocNo,
len(DocKg.hNodeId))
lObjId = DocKg.hNodeId.keys()
lObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in lObjId]
mhFeature = self.EdgeFeatureCenter.ExtractObjObjFeature(
lObj, query)
for FeatureName in self.EdgeFeatureCenter.FeatureDims():
OutDir = self.OutDir + '/' + FeatureName + '/' + qid
if not os.path.exists(OutDir):
os.makedirs(OutDir)
llEdgeFeatureScore = [[
hFeature[FeatureName] for hFeature in lhFeature
] for lhFeature in mhFeature]
DocKg.mEdgeMatrix = np.array(llEdgeFeatureScore)
DocKg.dump(OutDir + '/' + DocKg.DocNo)
logging.debug('[%s] feature for doc [%s] dummped', FeatureName,
DocKg.DocNo)
logging.info('[%s] dummped [%d] node', DocKg.DocNo, len(DocKg))
logging.info('[%s-%s] doc kg formed', qid, query)
return True
def ShowConf(cls):
print cls.__name__
FbObjCacheCenterC.ShowConf()
print 'objobjfeaturegroup'
ObjObjEdgeFeatureKGExtractorC.ShowConf()
ObjObjEdgeFeaturePreCalcSimExtractorC.ShowConf()
ObjObjEdgeFeatureTextSimExtractorC.ShowConf()
ObjObjEdgeFeatureTypeExtractorC.ShowConf()
ObjObjEdgeFeatureEmbSimExtractorC.ShowConf()
def ShowConf():
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'nodedir\nqobjfeaturegroup\ndocobjfeaturegroup\nobjobjfeaturegroup'
QueryObjEdgeFeatureAnaExtractorC.ShowConf()
DocObjEdgeFeatureFaccExtractorC.ShowConf()
ObjObjEdgeFeatureKGExtractorC.ShowConf()
ObjObjEdgeFeaturePreCalcSimExtractorC.ShowConf()
ObjObjEdgeFeatureTextSimExtractorC.ShowConf()
def ShowConf(cls):
cxBaseC.ShowConf()
print cls.__name__
print 'nodedir'
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
LeToRFeatureExtractCenterC.ShowConf()
FbQObjFeatureExtractCenterC.ShowConf()
FbObjDocFeatureExtractCenterC.ShowConf()
ObjObjFeatureExtractCenterC.ShowConf()
class LESRanker(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.Evaluator = AdhocEvaC()
self.Inferener = LESInferencerC()
self.QDocNodeDataDir = ""
self.OrigQWeight = 0.5
self.UseQObjOnly = True
@classmethod
def ShowConf(cls):
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
AdhocEvaC.ShowConf()
print 'qdocnodedatadir\norigqweight 0.5\nqobjonly 1'
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.Searcher.SetConf(ConfIn)
self.Evaluator.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
self.QDocNodeDataDir = self.conf.GetConf('qdocnodedatadir') + '/'
self.OrigQWeight = self.conf.GetConf('origqweight', self.OrigQWeight)
self.UseQObjOnly = bool(self.conf.GetConf('qobjonly', 1))
def LoadQDocObj(self, query):
InName = self.QDocNodeDataDir + IndriSearchCenterC.GenerateQueryTargetName(
query)
hQDocObj = {}
for line in open(InName):
key, ObjId = line.strip().split('\t')
if not key in hQDocObj:
hQDocObj[key] = [ObjId]
else:
hQDocObj[key].append(ObjId)
logging.info('query [%s] q doc obj [%d] loaded', query, len(hQDocObj))
return hQDocObj
def RankingForOneQ(self, qid, query):
logging.info('Start LES ranking for [%s-%s]', qid, query)
lDoc = self.Searcher.RunQuery(query, qid)
logging.info('doc fetched')
hQDocObj = self.LoadQDocObj(query)
QKey = 'q_%s' % (qid)
if not QKey in hQDocObj:
#do nothing
logging.info('query [%s] has no object, return raw raning', qid)
return [doc.DocNo for doc in lDoc]
lQObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in hQDocObj[QKey]]
lDocLESScore = []
LesCnt = 0
for doc in lDoc:
if self.UseQObjOnly:
lDocObj = lQObj
else:
if not doc.DocNo in hQDocObj:
lDocLESScore.append(0)
continue
lDocObj = [
self.ObjCenter.FetchObj(ObjId)
for ObjId in hQDocObj[doc.DocNo]
]
score = self.Inferener.inference(query, doc, lQObj, lDocObj)
if score != 0:
#if 0, means the obj has no desp (or very short one), doesn't count as valid score
LesCnt += 1
lDocLESScore.append(score)
#add average score to doc without annotation
#using zero is not very proper
AvgScore = sum(lDocLESScore) / float(LesCnt)
lDocLESScore = [
item if item != 0 else AvgScore for item in lDocLESScore
]
lScore= [self.OrigQWeight * math.exp(doc.score) + (1-self.OrigQWeight) * LESScore \
for doc,LESScore in zip(lDoc,lDocLESScore)]
lDocNoScore = zip([doc.DocNo for doc in lDoc], lScore)
lDocNoScore.sort(key=lambda item: item[1], reverse=True)
lRankedDocNo = [item[0] for item in lDocNoScore]
logging.info('query [%s] ranked', qid)
return lRankedDocNo
def Process(self, QIn, OutName):
lQidQuery = [
line.split('\t') for line in open(QIn).read().splitlines()
]
llDocNo = [self.RankingForOneQ(qid, query) for qid, query in lQidQuery]
logging.info('start evaluation')
lQid = [item[0] for item in lQidQuery]
lQuery = [item[1] for item in lQidQuery]
lPerQEvaRes = self.Evaluator.EvaluateFullRes(lQid, lQuery, llDocNo)
out = open(OutName, 'w')
for qid, EvaRes in lPerQEvaRes:
print >> out, qid + '\t' + EvaRes.dumps()
out.close()
logging.info('%s %s', lPerQEvaRes[-1][0], lPerQEvaRes[-1][1].dumps())
return True
what's my output:
add "\t name a \t name b" in the end of each line
'''
import site
site.addsitedir('/bos/usr0/cx/PyCode/cxPyLib')
site.addsitedir('/bos/usr0/cx/PyCode/GoogleAPI')
from ObjCenter.FbObjCacheCenter import FbObjCacheCenterC
from cxBase.Conf import cxConfC
import sys
if 2 != len(sys.argv):
FbObjCacheCenterC.ShowConf()
print "in\nout"
sys.exit()
ObjCenter = FbObjCacheCenterC(sys.argv[1])
conf = cxConfC(sys.argv[1])
InName = conf.GetConf('in')
OutName = conf.GetConf('out')
out = open(OutName, 'w')
cnt = 0
for line in open(InName):
line = line.strip()
vCol = line.split('\t')
lName = [ObjCenter.FetchObjName(ObjId) for ObjId in vCol[:2]]
def ShowConf(cls):
cxBaseC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'origqweight 0.5'
class LESRanker(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.ObjCenter = FbObjCacheCenterC()
self.Inferener = LESInferencerC()
self.DocKgDir = ""
self.hQObj = {}
self.OrigQWeight = 0.5
@classmethod
def ShowConf(cls):
cxBaseC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'origqweight 0.5'
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.DocKgDir = self.conf.GetConf('dockgdir')
QAnaInName = self.conf.GetConf('qanain')
self.LoadQObj(QAnaInName)
self.ObjCenter.SetConf(ConfIn)
self.OrigQWeight = self.conf.GetConf('origqweight', self.OrigQWeight)
def LoadQObj(self,QAnaInName):
for line in open(QAnaInName).read().splitlines():
vCol = line.strip().split('\t')
qid = vCol[0]
ObjId = vCol[2]
score = vCol[-1]
if not qid in self.hQObj:
self.hQObj[qid] = [[ObjId,score]]
else:
self.hQObj[qid].append([ObjId,score])
logging.info('qobj loaded from [%s]',QAnaInName)
return True
def RankScoreForDoc(self,qid,query,doc):
DocKg = SearchResDocGraphConstructorC.LoadDocGraph(self.DocKgDir, qid, doc.DocNo)
lQObjId = [item[0] for item in self.hQObj[qid]]
lDocObjId = DocKg.hNodeId.keys()
lQObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in lQObjId]
lDocObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in lDocObjId]
score = self.Inferener.inference(query, doc, lQObj, lDocObj)
return score
def Rank(self,qid,query,lDoc):
if not qid in self.hQObj:
logging.warn('qid [%s] no ana obj, withdraw to given score',qid)
return [doc.DocNo for doc in lDoc]
lScore = [self.RankScoreForDoc(qid, query, doc) for doc in lDoc]
lDocNoScore = zip([doc.DocNo for doc in lDoc],lScore)
lDocNoScore.sort(key=lambda item: item[1], reverse = True)
lRankRes = [item[0] for item in lDocNoScore]
return lRankRes
def ShowConf():
SearchResDocGraphConstructorC.ShowConf()
ObjObjFeatureExtractCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
class BoeLmWeighterC(BoeLmC):
def __init__(self,ConfIn = ""):
self.Init()
if "" != ConfIn:
self.SetConf(ConfIn)
def Init(self):
BoeLmC.Init(self)
self.DocTextDir = ""
self.ObjCenter = FbObjCacheCenterC()
self.CtfCenter = TermCtfC()
self.lInferenceWeight = [1,0,0]
self.hDocText = {}
def SetConf(self,ConfIn):
conf = cxConfC(ConfIn)
self.DocTextDir = conf.GetConf('doctextdir')
self.LoadDocText()
self.ObjCenter.SetConf(ConfIn)
CtfInName = conf.GetConf('objctf')
self.CtfCenter.Load(CtfInName)
@classmethod
def ShowConf():
print 'doctextdir\nobjctf'
FbObjCacheCenterC.ShowConf()
def LoadDocText(self):
for fname in WalkDir(self.DocTextDir):
for line in open(fname):
DocNo,text = line.strip().split('\t')
self.hDocText[DocNo] = text
logging.info('doc text loaded')
def GetAllIdf(self,DocKg):
lItem = DocKg.hNodeId.items()
lItem.sort(key=lambda item:item[1])
lObjId = [item[0] for item in lItem]
lRes = []
for ObjId in lObjId:
idf = self.CtfCenter.GetLogIdf(ObjId)
lRes.append(idf)
return lRes
def GetAllTf(self,DocKg):
return list(DocKg.vNodeWeight)
def GetAllTextCosine(self,DocKg):
DocText = ""
if DocKg.DocNo in self.hDocText:
DocText = self.hDocText[DocKg.DocNo]
lCos = []
if "" == DocText:
return [0] * len(DocKg)
DocLm = LmBaseC(DocText)
lItem = DocKg.hNodeId.items()
lItem.sort(key=lambda item:item[1])
lObjId = [item[0] for item in lItem]
for ObjId in lObjId:
desp = self.ObjCenter.FetchObjDesp(ObjId)
lm = LmBaseC(desp)
lCos.append(LmBaseC.Cosine(lm, DocLm))
return lCos
def GetTextCosine(self,ObjId,DocKg):
DocText = ""
if DocKg.DocNo in self.hDocText:
DocText = self.hDocText[DocKg.DocNo]
DocLm = LmBaseC(DocText)
desp = self.ObjCenter.FetchObjDesp(ObjId)
lm = LmBaseC(desp)
score = LmBaseC.Cosine(lm, DocLm)
if 0 == score:
return self.MinLogProb
return math.log(score)
def LinearWeightTfIdfTextSim(self,ObjId,DocKg,TfScore = 1,IdfScore = 0, TextSimScore = 0):
if not ObjId in DocKg:
return self.MinLogProb
lTf = np.zeros(len(DocKg))
lIdf = np.zeros(len(DocKg))
if TfScore != 0:
lTf = np.array(self.GetAllTf(DocKg))
if IdfScore != 0:
lIdf = np.array(self.GetAllIdf(DocKg))
# lCos = np.array(self.GetAllTextCosine(DocKg))
TextSim = 0
if TextSimScore != 0:
TextSim = self.GetTextCosine(ObjId,DocKg)
W = np.array([TfScore,IdfScore,TextSimScore])
W = W / float(sum(W))
lScore = lTf * W[0] + lIdf * W[1]
res = self.MinLogProb * (W[0] + W[1])
if ObjId in DocKg:
p = DocKg.hNodeId[ObjId]
res = lScore[p]
res = res + TextSim * TextSimScore
return res
def inference(self, ObjId, DocKg):
return self.LinearWeightTfIdfTextSim(ObjId, DocKg, self.lInferenceWeight[0], self.lInferenceWeight[1], self.lInferenceWeight[2])
def Init(self):
cxBaseC.Init(self)
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.NeighborNum = 50
class GraphFullFeatureExtractCenterC(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.NodeDir = ""
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.QDocFeatureExtractor = LeToRFeatureExtractCenterC()
self.QObjFeatureExtractor = FbQObjFeatureExtractCenterC()
self.DocObjFeatureExtractor = FbObjDocFeatureExtractCenterC()
self.ObjObjFeatureExtractor = ObjObjFeatureExtractCenterC()
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.NodeDir = self.conf.GetConf('nodedir') + '/'
self.Searcher.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
self.QDocFeatureExtractor.SetConf(ConfIn)
self.QObjFeatureExtractor.SetConf(ConfIn)
self.DocObjFeatureExtractor.SetConf(ConfIn)
self.ObjObjFeatureExtractor.SetConf(ConfIn)
logging.info('graph full feature extractor conf setted')
@classmethod
def ShowConf(cls):
cxBaseC.ShowConf()
print cls.__name__
print 'nodedir'
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
LeToRFeatureExtractCenterC.ShowConf()
FbQObjFeatureExtractCenterC.ShowConf()
FbObjDocFeatureExtractCenterC.ShowConf()
ObjObjFeatureExtractCenterC.ShowConf()
def FormulateNodes(self, qid, query):
'''
get ldoc and read lObjId
fill lObjId
'''
logging.info('formulating node for q [%s][%s]', qid, query)
lDoc = self.Searcher.RunQuery(query, qid)
lDocNo, lQObjId, llDocObjId = NodeCollectorCenterC.LoadRawFormatNodeRes(
query, self.NodeDir)
#match lDoc dim lDocNo dim
lDoc = IndriSearchCenterC.RearrangeDocOrder(lDoc, lDocNo)
lQObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in lQObjId]
llDocObj = [[self.ObjCenter.FetchObj(ObjId) for ObjId in lDocObjId]
for lDocObjId in llDocObjId]
while len(llDocObj) < len(lDoc):
#add empty list for docs have no objects (thus will restrict to EsdRank)
#if lQObj is also empty, then it is LeToR
llDocObj.append([])
logging.info('q[%s] all node fetched, q node %s', qid,
json.dumps([Obj.GetId() for Obj in lQObj]))
return lDoc, lQObj, llDocObj
def Process(self, qid, query, OutDir):
'''
'''
lDoc, lQObj, llDocObj = self.FormulateNodes(qid, query)
for doc, lDocObj in zip(lDoc, llDocObj):
hQDocFeature, lhQObjFeature, lhDocObjFeature, llhObjObjFeature = self.ExtractFeatureForOneQDoc(
qid, query, doc, lQObj + lDocObj)
self.DumpPerQRes(qid, query, doc, lQObj + lDocObj, hQDocFeature,
lhQObjFeature, lhDocObjFeature, llhObjObjFeature,
OutDir)
logging.info('q [%s] processed')
return True
def PipeRun(self, QInName, OutDir):
lQidQuery = [
line.split('\t') for line in open(QInName).read().splitlines()
]
for qid, query in lQidQuery:
self.Process(qid, query, OutDir)
logging.info('queries in [%s] processed features at [%s]', QInName,
OutDir)
return True
def ExtractFeatureForOneQDoc(self, qid, query, doc, lObj):
#if wanna speed up, cache features
#for clearity, now just extract multiple times
hQDocFeature = self.QDocFeatureExtractor.Process(qid, query, doc)
logging.debug('q[%s][%s] ltr feature extracted', query, doc.DocNo)
lhQObjFeature = self.QObjFeatureExtractor.ProcessOneQuery([qid, query],
lObj)
logging.debug('q[%s][%s] obj feature extracted', query, doc.DocNo)
lhDocObjFeature = self.DocObjFeatureExtractor.ProcessOneQueryDocPair(
[qid, query], doc, lObj)
logging.debug('q[%s][%s] doc obj feature extracted', query, doc.DocNo)
llhObjObjFeature = self.ObjObjFeatureExtractor.Process(
qid, query, lObj) #symetric matrix
logging.debug('q[%s] [%s] obj obj feature extracted', query, doc.DocNo)
logging.debug('q [%s][%s] all doc graph feature extracted', query,
doc.DocNo)
return hQDocFeature, lhQObjFeature, lhDocObjFeature, llhObjObjFeature
def DumpPerQRes(self, qid, query, doc, lObj, hQDocFeature, lhQObjFeature,
lhDocObjFeature, llhObjObjFeature, OutDir):
'''
raw:
a dir for this q
a file for each doc
node a, node b, hFeature.json
'''
if not os.path.exists(OutDir + '/' + qid):
os.makedirs(OutDir + '/' + qid)
OutName = OutDir + '/' + qid + '/' + doc.DocNo
out = open(OutName, 'w')
#q doc
print >> out, 'q_%s' % (qid) + '\t' + doc.DocNo + '\t' + json.dumps(
hQDocFeature)
#obj doc
for Obj, hDocObjFeature in zip(lObj, lhDocObjFeature):
print >> out, Obj.GetId() + '\t' + doc.DocNo + '\t' + json.dumps(
hDocObjFeature)
#q obj
for Obj, hQObjFeature in zip(lObj, lhQObjFeature):
print >> out, 'q_%s' % (
qid) + '\t' + Obj.GetId() + '\t' + json.dumps(hQObjFeature)
print >> out, Obj.GetId() + '\t' + 'q_%s' % (
qid) + '\t' + json.dumps(hQObjFeature)
#make it symmetric
#obj obj
for i in range(len(lObj)):
for j in range(len(lObj)):
if i == j:
continue
print >> out, lObj[i].GetId() + '\t' + lObj[j].GetId(
) + '\t' + json.dumps(llhObjObjFeature[i][j])
logging.info('q[%s] doc [%s] graph dumped to file [%s]', qid,
doc.DocNo, OutName)
return True
columns are the same as original lines,
but with an additional column saying which edge it is
'''
import site
site.addsitedir('/bos/usr0/cx/PyCode/cxPyLib')
site.addsitedir('/bos/usr0/cx/PyCode/SemanticSearch')
site.addsitedir('/bos/usr0/cx/PyCode/GoogleAPI')
from ObjCenter.FbObjCacheCenter import FbObjCacheCenterC
from cxBase.Conf import cxConfC
import sys
if 2 != len(sys.argv):
FbObjCacheCenterC.ShowConf()
print 'in\nout'
sys.exit()
conf = cxConfC(sys.argv[1])
CacheCenter = FbObjCacheCenterC(sys.argv[1])
InName = conf.GetConf('in')
OutName = conf.GetConf('out')
out = open(OutName, 'w')
for line in open(InName):
line = line.strip()
vCol = line.split('\t')
ObjId = vCol[2]
def ShowConf():
print 'doctextdir\nobjctf'
FbObjCacheCenterC.ShowConf()
class EdgeFeatureExtractCenterC(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.lQObjFeatureGroup = []
self.lObjObjFeatureGroup = []
self.lDocObjFeatureGroup = []
self.QObjAnaExtractor = QueryObjEdgeFeatureAnaExtractorC()
self.DocObjFaccExtractor = DocObjEdgeFeatureFaccExtractorC()
self.ObjObjKGExtractor = ObjObjEdgeFeatureKGExtractorC()
self.ObjObjPreCalcExtractor = ObjObjEdgeFeaturePreCalcSimExtractorC()
self.ObjObjTextSimExtractor = ObjObjEdgeFeatureTextSimExtractorC()
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.NodeDir = ""
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.Searcher.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
self.NodeDir = self.conf.GetConf('nodedir') + '/'
self.lQObjFeatureGroup = self.conf.GetConf('qobjfeaturegroup',
self.lQObjFeatureGroup)
self.lDocObjFeatureGroup = self.conf.GetConf('docobjfeaturegroup',
self.lDocObjFeatureGroup)
self.lObjObjFeatureGroup = self.conf.GetConf('objobjfeaturegroup',
self.lObjObjFeatureGroup)
if 'ana' in self.lQObjFeatureGroup:
self.QObjAnaExtractor.SetConf(ConfIn)
if 'facc' in self.lDocObjFeatureGroup:
self.DocObjFaccExtractor.SetConf(ConfIn)
if 'kg' in self.lObjObjFeatureGroup:
self.ObjObjKGExtractor.SetConf(ConfIn)
if 'precalc' in self.lObjObjFeatureGroup:
self.ObjObjPreCalcExtractor.SetConf(ConfIn)
if 'textsim' in self.lObjObjFeatureGroup:
self.ObjObjTextSimExtractor.SetConf(ConfIn)
logging.info('edge feature center confs setted')
@staticmethod
def ShowConf():
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'nodedir\nqobjfeaturegroup\ndocobjfeaturegroup\nobjobjfeaturegroup'
QueryObjEdgeFeatureAnaExtractorC.ShowConf()
DocObjEdgeFeatureFaccExtractorC.ShowConf()
ObjObjEdgeFeatureKGExtractorC.ShowConf()
ObjObjEdgeFeaturePreCalcSimExtractorC.ShowConf()
ObjObjEdgeFeatureTextSimExtractorC.ShowConf()
def FormulateNodes(self, qid, query):
'''
get ldoc and read lObjId
fill lObjId
'''
logging.info('formulating node for q [%s][%s]', qid, query)
lDoc = self.Searcher.RunQuery(query, qid)
lObjId = open(self.NodeDir +
IndriSearchCenterC.GenerateQueryTargetName(query)).read(
).splitlines()
lObj = [self.ObjCenter.FetchObj(ObjId) for ObjId in lObjId]
logging.info('q[%s] [%d] doc [%d] obj', query, len(lDoc), len(lObj))
return lDoc, lObj
def ExtractPerQObj(self, qid, query, obj):
hFeature = {}
logging.debug('start extracting q[%s]-obj[%s] feature', query,
obj.GetId())
if 'ana' in self.lQObjFeatureGroup:
hFeature.update(self.QObjAnaExtractor.process(qid, query, obj))
logging.debug('q[%s]-obj[%s] feature extracted', query, obj.GetId())
return hFeature
def ExtractQObjFeature(self, qid, query, lObj):
lhFeature = []
logging.info('start extracting [%s][%s] q-obj feature [%d] obj', qid,
query, len(lObj))
for obj in lObj:
hFeature = self.ExtractPerQObj(qid, query, obj)
lhFeature.append(hFeature)
logging.info('q obj feature extracted')
return lhFeature
def ExtractPerDocObj(self, doc, obj):
hFeature = {}
logging.debug('start extracting doc[%s]-obj[%s] feature', doc.DocNo,
obj.GetId())
if 'facc' in self.lDocObjFeatureGroup:
hFeature.update(self.DocObjFaccExtractor.process(doc, obj))
logging.debug('doc[%s]-obj[%s] feature extracted', doc.DocNo,
obj.GetId())
return hFeature
def ExtractDocObjFeature(self, lDoc, lObj):
llhFeature = [] #doc \times obj
logging.info('start extract [%d] doc - [%d] obj feature mtx',
len(lDoc), len(lObj))
for doc in lDoc:
lhFeature = []
for obj in lObj:
hFeature = self.ExtractPerDocObj(doc, obj)
lhFeature.append(hFeature)
llhFeature.append(lhFeature)
logging.info('doc obj feature extracted')
return llhFeature
def ExtractPerObjObj(self, ObjA, ObjB, query):
hFeature = {}
logging.debug('start extracting for obj pair [%s-%s]', ObjA.GetId(),
ObjB.GetId())
if 'kg' in self.lObjObjFeatureGroup:
hFeature.update(self.ObjObjKGExtractor.process(ObjA, ObjB))
if 'precalc' in self.lObjObjFeatureGroup:
hFeature.update(
self.ObjObjPreCalcExtractor.process(ObjA, ObjB, query))
if 'textsim' in self.lObjObjFeatureGroup:
hFeature.update(self.ObjObjTextSimExtractor.process(ObjA, ObjB))
logging.debug('obj pair [%s-%s] feature extracted', ObjA.GetId(),
ObjB.GetId())
return hFeature
def ExtractObjObjFeature(self, lObj, query):
llhFeature = [] #obj -> obj, diagonal is empty
logging.info('start extract [%d] obj pair feature mtx', len(lObj))
for ObjA in lObj:
lhFeature = []
for ObjB in lObj:
if ObjA.GetId() == ObjB.GetId():
continue
hFeature = self.ExtractPerObjObj(ObjA, ObjB, query)
lhFeature.append(hFeature)
llhFeature.append(lhFeature)
logging.info('obj obj feature extracted')
return llhFeature
def Process(self, qid, query):
lDoc, lObj = self.FormulateNodes(qid, query)
logging.info('nodes fetched')
lQObjFeature = self.ExtractQObjFeature(qid, query, lObj)
llDocObjFeature = self.ExtractDocObjFeature(lDoc, lObj)
llObjObjFeature = self.ExtractObjObjFeature(lObj, query)
return lDoc, lObj, lQObjFeature, llDocObjFeature, llObjObjFeature
def DumpRes(self, OutName, query, lDoc, lObj, lQObjFeature,
llDocObjFeature, llObjObjFeature):
out = open(OutName, 'w')
for obj, hFeature in zip(lObj, lQObjFeature):
print >> out, query + '\t' + obj.GetId() + '\t' + json.dumps(
hFeature)
for doc, lhFeature in zip(lDoc, llDocObjFeature):
for obj, hFeature in zip(lObj, lhFeature):
print >> out, doc.DocNo + '\t' + obj.GetId(
) + '\t' + json.dumps(hFeature)
for ObjA, lhFeature in zip(lObj, llObjObjFeature):
for ObjB, hFeature in zip(lObj, lhFeature):
print >> out, ObjA.GetId() + '\t' + ObjB.GetId(
) + '\t' + json.dumps(hFeature)
out.close()
logging.info('query [%s] feature dumped', query)
def PipeRun(self, QInName, OutDir):
'''
for now:
output raw type
each file is a query's edge features
each line is query|doc|obj \t obj \t json.dumps(hFeature)
'''
lQidQuery = [
line.split('\t') for line in open(QInName).read().splitlines()
]
for qid, query in lQidQuery:
logging.info('start extracting for [%s][%s]', qid, query)
lDoc, lObj, lQObjFeature, llDocObjFeature, llObjObjFeature = self.Process(
qid, query)
OutName = OutDir + '/' + IndriSearchCenterC.GenerateQueryTargetName(
query)
logging.info('[%s][%s] extracted, dumpping to [%s]', qid, query,
OutName)
self.DumpRes(OutName, query, lDoc, lObj, lQObjFeature,
llDocObjFeature, llObjObjFeature)
logging.info('all finished')
return
def ShowConf():
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'neighbornum'
class EntityCorrelationFromTextSimC(cxBaseC):
def Init(self):
cxBaseC.Init(self)
self.Searcher = IndriSearchCenterC()
self.ObjCenter = FbObjCacheCenterC()
self.NeighborNum = 50
def SetConf(self, ConfIn):
cxBaseC.SetConf(self, ConfIn)
self.Searcher.SetConf(ConfIn)
self.ObjCenter.SetConf(ConfIn)
self.NeighborNum = self.conf.GetConf('neighbornum', self.NeighborNum)
@staticmethod
def ShowConf():
cxBaseC.ShowConf()
IndriSearchCenterC.ShowConf()
FbObjCacheCenterC.ShowConf()
print 'neighbornum'
def ProcessOneObj(self, ObjId, name):
'''
return lObjNeighbor=[objid,KL score] top self.NeighborNum
'''
#search in index, get top 1000
query = TextBaseC.RawClean(name)
if "" == query:
return []
lObjDoc = self.Searcher.RunQuery(query)
lObjNeighbor = []
ThisDesp = self.ObjCenter.FetchObjDesp(ObjId)
ThisLm = LmBaseC(ThisDesp)
ThisVec = VectorC(ThisLm.hTermTF)
# print '[%s:%s] desp : [%s]' %(ObjId,name,ThisDesp)
if len(ThisLm.hTermTF) == 0:
return []
for ObjDoc in lObjDoc:
Id = ObjDoc.DocNo
if Id == ObjId:
continue
if not Id.startswith('/m/'):
print "[%s %s] neighbor id [%s] format error" % (ObjId, name,
Id)
continue
# print "get neighbor [%s] [%s]" %(Id,ObjDoc.GetContent())
# NeighborDesp = ObjDoc.GetContent()
NeighborLm = LmBaseC(ObjDoc)
NeighborVec = VectorC(NeighborLm.hTermTF)
if len(NeighborVec.hDim) == 0:
continue
score = VectorC.KL(ThisVec, NeighborVec)
lObjNeighbor.append([Id, -score])
# print "[%s %s] KL [%f]" %(ObjId,Id,score)
# print "%s\n%s" %(json.dumps(ThisVec.hDim),json.dumps(NeighborVec.hDim))
lObjNeighbor.sort(key=lambda item: item[1], reverse=True)
print "[%s:%s] neighbor id score get" % (ObjId, name)
return lObjNeighbor
def Process(self, ObjInName, OutName):
out = open(OutName, 'w')
for line in open(ObjInName):
vCol = line.strip().split('\t')
if len(vCol) < 2:
continue
lObjNeighbor = self.ProcessOneObj(vCol[0], vCol[1])
for NeighborId, score in lObjNeighbor[:self.NeighborNum]:
print >> out, '%s\t%s\t%f\t%s\t%s' % (
vCol[0], NeighborId, score, vCol[1],
self.ObjCenter.FetchObjName(NeighborId))
print "[%s:%s] done" % (vCol[0], vCol[1])
out.close()
print "finished"
def Init(self):
SearchResDocGraphConstructorC.Init(self)
self.EdgeFeatureCenter = ObjObjFeatureExtractCenterC()
self.ObjCenter = FbObjCacheCenterC()