Python fetch示例

示例#1

文件： test_crawler.py 项目： Faith4444/MSpider

#!/usr/bin/python
#-*-coding:utf-8-*-
#Author : Manning
#Date : 2015-10-17
import sys
sys.path.append(sys.path[0].split('MSpider')[0] + "MSpider")
from lib.core.crawl import crawl
from lib.core.fetch import fetch


if __name__ == '__main__':
    url = 'http://www.wooyun.org/bugs/'
    html = fetch(url)
    for i in crawl(url,html):
        print i

示例#2

#!/usr/bin/python
#-*-coding:utf-8-*-
#Author : Manning
#Date : 2015-10-17
import sys
sys.path.append(sys.path[0].split('MSpider')[0] + "MSpider")
from lib.core.crawl import crawl
from lib.core.fetch import fetch

if __name__ == '__main__':
    url = 'http://www.wooyun.org/bugs/'
    html = fetch(url)
    for i in crawl(url, html):
        print i

示例#3

#!/usr/bin/python
#-*-coding:utf-8-*-
#Author : Manning
#Date : 2015-10-17
import sys
sys.path.append(sys.path[0].split('MSpider')[0] + "MSpider")
from lib.core.fetch import fetch

if __name__ == '__main__':
    url = 'http://www.baidu.com'
    print fetch(url)

示例#4

文件： test_fetcher.py 项目： Faith4444/MSpider

#!/usr/bin/python
#-*-coding:utf-8-*-
#Author : Manning
#Date : 2015-10-17
import sys
sys.path.append(sys.path[0].split('MSpider')[0] + "MSpider")
from lib.core.fetch import fetch


if __name__ == '__main__':
    url = 'http://www.baidu.com'
    print fetch(url)