Python SYMetalNetSpiderUtils.returnStartContext Examples

Example #1

File: SYMetalNetSpider.py Project: wusezhangserver/kttspider

def crawDailyMetalComments(link, webNet):
    startContext = SYMetalNetSpiderUtils.returnStartContext(link, '<tbody>')
    currentList = []
    i = 0
    while i < 8:
        targetContext = SYMetalNetSpiderUtils.divisionTarget(
            startContext, '<tr>', '</tr>')
        startContext = targetContext['nextContext']
        currentContext = targetContext['targetContext']
        time = SYMetalNetSpiderUtils.filterContextByTarget(
            currentContext, '<td>', '</td>')
        nextContext = SYMetalNetSpiderUtils.filterAfterContext(
            currentContext, '</td>')
        pubDate = SYMetalNetSpiderUtils.filterContextByTarget(
            nextContext, '<td>', '</td>')
        pubDate = SYMetalNetSpiderUtils.removeSpecialCharacter(pubDate)
        nextContext = SYMetalNetSpiderUtils.filterAfterContext(
            nextContext, '</td>')
        linkUrl = webNet + SYMetalNetSpiderUtils.filterContextByTarget(
            nextContext, '<a href="', '">')
        title = SYMetalNetSpiderUtils.filterContextByTarget(
            nextContext, '">', '</a>')
        title = SYMetalNetSpiderUtils.removeSpecialCharacter(title)
        descriptContext = crawDescriptContext(linkUrl)
        currentList.append([
            str(uuid.uuid1()), linkUrl, title, pubDate, descriptContext,
            'METAL', 'SYNET'
        ])
        i += 1
    return currentList

Example #2

File: SYMetalNetSpider.py Project: wusezhangserver/kttspider

def crawDescriptContext(linkUrl):
    startContext = SYMetalNetSpiderUtils.returnStartContext(
        linkUrl, '<div class="Nrr_middle">')
    filterContext = SYMetalNetSpiderUtils.filterAfterContext(
        startContext, '<br><br>')
    filterContext = SYMetalNetSpiderUtils.filterContextByTarget(
        filterContext, '', '<br>')
    return filterContext

Example #3

File: SYMetalNetSpider.py Project: nicevoice/SpiderKTServer

def crawDailyMetalComments(link, webNet):
    startContext = SYMetalNetSpiderUtils.returnStartContext(link, "<tbody>")
    currentList = []
    i = 0
    while i < 8:
        targetContext = SYMetalNetSpiderUtils.divisionTarget(startContext, "<tr>", "</tr>")
        startContext = targetContext["nextContext"]
        currentContext = targetContext["targetContext"]
        time = SYMetalNetSpiderUtils.filterContextByTarget(currentContext, "<td>", "</td>")
        nextContext = SYMetalNetSpiderUtils.filterAfterContext(currentContext, "</td>")
        pubDate = SYMetalNetSpiderUtils.filterContextByTarget(nextContext, "<td>", "</td>")
        pubDate = SYMetalNetSpiderUtils.removeSpecialCharacter(pubDate)
        nextContext = SYMetalNetSpiderUtils.filterAfterContext(nextContext, "</td>")
        linkUrl = webNet + SYMetalNetSpiderUtils.filterContextByTarget(nextContext, '<a href="', '">')
        title = SYMetalNetSpiderUtils.filterContextByTarget(nextContext, '">', "</a>")
        title = SYMetalNetSpiderUtils.removeSpecialCharacter(title)
        descriptContext = crawDescriptContext(linkUrl)
        currentList.append([str(uuid.uuid1()), linkUrl, title, pubDate, descriptContext, "METAL", "SYNET"])
        i += 1
    return currentList

Example #4

File: SYMetalNetSpider.py Project: nicevoice/SpiderKTServer

def crawDescriptContext(linkUrl):
    startContext = SYMetalNetSpiderUtils.returnStartContext(linkUrl, '<div class="Nrr_middle">')
    filterContext = SYMetalNetSpiderUtils.filterAfterContext(startContext, "<br><br>")
    filterContext = SYMetalNetSpiderUtils.filterContextByTarget(filterContext, "", "<br>")
    return filterContext