Python Crawler._create_spider Examples

Example #1

File: test_for_engine_01.py Project: lalacat/crawler

from test.framework.spider import Test_Spider_2
from test.framework.spider.test_spider.test_Spider_03 import Test_Spider_3
from test.framework.setting import Setting
from test.framework.core.crawler import Crawler
from test.framework.spider.test_spider.test_Spider_01 import Test_Spider_1
from twisted.internet import reactor, defer
import logging
LOG_FORMAT = '%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s'
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt=DATE_FORMAT)


def finish_crawl(content):
    logging.info("finish")
    return content


settings = Setting()
crawler_01 = Crawler(Test_Spider_1, settings)
crawler_02 = Crawler(Test_Spider_2, settings)
crawler_03 = Crawler(Test_Spider_3, settings)
spider_01 = crawler_01._create_spider()
spider_02 = crawler_02._create_spider()
spider_03 = crawler_03._create_spider()
c1 = crawler_01.crawl()
c2 = crawler_02.crawl()
c3 = crawler_03.crawl()
dd = defer.DeferredList([c1, c2, c3])
dd.addBoth(lambda _: reactor.stop())
reactor.run()

Example #2

def request_errback(content):
    print("request_and_response errback")
    print(content[1])
    return content

def agent_print(content):
    print("agent_print")
    print(type(content))
    print(content)

request = Request(url=url,callback=request_callback,method='get',
                  headers=headers,errback=request_errback,meta={"download_timeout":2})

settings = Setting()
crawler = Crawler(LJSpider,settings)
spider = crawler._create_spider()
downloader = Downloader(crawler)

"""
httphandler = HTTPDownloadHandler(settings)
agent = httphandler.download_request(request,spider)
agent.addCallback(agent_print)
agent.addErrback(request_errback)
"""
agent = downloader.fetch(request,spider)
agent.addCallback(request_callback)
agent.addBoth(lambda _: reactor.stop())

reactor.run()

Example #3

File: test_crawl_03_lianjia_allzone_xiaoqu.py Project: lalacat/crawler

import logging
LOG_FORMAT = '%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s'
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt=DATE_FORMAT)

# mongodb服务的地址和端口号
mongo_url = "127.0.0.1:27017"

# 连接到mongodb，如果参数不填，默认为“localhost:27017”
client = pymongo.MongoClient(mongo_url)

#连接到数据库myDatabase
DATABASE = "LianJia"
db = client[DATABASE]

#连接到集合(表):myDatabase.myCollection
COLLECTION = "XiaoQu"
db_coll = db[COLLECTION]

projectionFields = {'_id': False}  # 用字典指定
queryArgs = {"total_zone_name": "pudong"}

searchRes = db_coll.find(queryArgs, projectionFields)
scheduler = searchRes.next()

settings = Setting()
crawler_01 = Crawler(Part_Zone, settings)
crawler_01._create_spider()
c1 = crawler_01.crawl()
c1.addBoth(lambda _: reactor.stop())
reactor.run()