示例#1
0
 def weightLinksOnPage( self, bodySoup, winSize = 3 ):
     #print self.kw.words
     links= {}
     ce = ContentExtractor()
     mc = ce.extractMappedContentList(bodySoup)
     for i in mc[0]:
         if type(i) != types.IntType:
             continue
         link = mc[1][i].get('href')
         links[link] = self.weightWindowed( mc[0], i, winSize ) # max(...)
     return links
示例#2
0
def process(pagedirs, contentdb):
    sources = os.listdir(pagedirs)
    ce = ContentExtractor(db=DB(contentdb))
    for source in sources:
        sourcedir = os.path.join(pagedirs, source)
        if not os.path.isdir(sourcedir):
            continue
        pagenames = os.listdir(sourcedir)
        pagenames = diff_task(pagenames, contentdb)
        for pagename in pagenames:
            hashurl = pagename
            pagename = os.path.join(sourcedir, pagename)
            with open(pagename, 'r') as f:
                ce.parse(page=f.read(), source=source, hashurl=hashurl)
示例#3
0
文件: main.py 项目: AugustLONG/zufang
def process(pagedirs, contentdb):
    sources = os.listdir(pagedirs)
    ce = ContentExtractor(
        db=DB(contentdb)
    )
    for source in sources:
        sourcedir = os.path.join(pagedirs, source)
        if not os.path.isdir(sourcedir):
            continue
        pagenames = os.listdir(sourcedir)
        pagenames = diff_task(pagenames, contentdb)
        for pagename in pagenames:
            hashurl = pagename
            pagename = os.path.join(sourcedir, pagename)
            with open(pagename, 'r') as f:
                ce.parse(
                    page=f.read(),
                    source=source,
                    hashurl=hashurl
                )