sys.path.append('../common') from common.logger import log import getdoc from database import table,tablemerge,dbconfig from aggregate.rmduplicate import Depository import time import logging from common import toolpit from config import merge1_rmd_file oldtime=time.time() # depos=Depository(0.8,merge1_rmd_file) # for debug depos=Depository(0.8) docs=getdoc.get_records_dayago(dbconfig.mergetable,20) for doc,summary in docs.iteritems(): depos.add_doc(doc, summary) # just add doc into the repository msg='Depository summary builds,time cost: %.2f (s)' % (time.time()-oldtime,) print msg logging.info(msg) def __addDoctoTable(doc): rows=table.getRecordsById(doc.source, doc.uid) if rows==-1: print '%s getRecordsById error'%(doc.source) return if len(rows[0])>0: exrecord=list(rows[0]) mtype,click='',0 exrecord+=[mtype,click] tablemerge.InsertItem(dbconfig.mergetable, exrecord)
import sys sys.path.append('..') sys.path.append('../common') from common.logger import log import getdoc2 from database import tablemerge,tablemerge2,dbconfig from aggregate.rmduplicate import Depository import time import logging from config import merge2_rmd_file oldtime=time.time() # depos=Depository(0.8,merge2_rmd_file) depos=Depository(0.8) docs=getdoc2.get_records_dayago(20) for doc,title in docs.iteritems(): depos.add_doc(doc, title) # just add doc into the repository msg='Depository title builds,time cost: %.2f (s)' % (time.time()-oldtime,) print msg logging.info(msg) def __addDoctoTable(doc): rows=tablemerge.getRecordsByNewsid(dbconfig.mergetable, doc.uid) if rows==-1: print 'tablemerge getRecordsById error' return if len(rows[0])>0: exrecord=list(rows[0][:1]+rows[0][2:-1]) related='' exrecord+=[related,] tablemerge2.InsertItem(dbconfig.mergetable2, exrecord) def __addDoctoRelated(exist_doc,doc):