示例#1
0
def crawl_company_info():
    spider_list = [
        List51jobSpider,
        List51SoleSpider,
        List56YeSpider,
        List58Spider,
        List8671Spider,
        List88152Spider,
        List99114Spider,
        ListCnlinfo88Spider,
        ListCnlistSpider,
        ListGanjiSpider,
        ListHuangye88Spider,
        ListMakepoloSpider,
        ListQincaiSpider,
        ListQiyeSpider,
        ListQy6Spider,
        ListSoudhSpider,
        ListYnshangjiSpider,
        ListYouboySpider,
        ListZhaopinSpider,
        ListZhaoshang100Spider,
        DetailTianyanchaSpider,  # 会被封
        Detail58Spider,
        DetailSzmqsSpider,
    ]

    run_scrapy_spider(spider_list)
示例#2
0
def crawl_new_proxy():
    spider_list = [
        # GoubanjiaSpider,
        KuaidailiSpider,
        XicidailiSpider,
        # NianshaoSpider,  # 包含国外IP
        # Ip181Spider,  # 包含国外IP
        # Ip3366Spider,  # 包含国外IP
    ]

    run_scrapy_spider(spider_list)
示例#3
0
def crawl_mobile_info():
    run_scrapy_spider([
        CnmoSpider,
    ])
示例#4
0
def crawl_zhixing_dlm():
    run_scrapy_spider(ShixinDLMSpider)
示例#5
0
def crawl_wenshu_info():
    run_scrapy_spider(WenshuPcSpider)
示例#6
0
def crawl_zhixing_court():
    run_scrapy_spider(ZhixingCourtSpider)
示例#7
0
def crawl_shixin_p2p():
    run_scrapy_spider([
        ShixinKaikaidaiSpider,
        ShixinMy089Spider,
    ])
示例#8
0
def crawl_shixin_other():
    run_scrapy_spider([
        ShixinCourtSpider,
        ShixinmingdanSpider,
        ShixinDLMSpider,
    ])
示例#9
0
def crawl_shixin_kuaicha():
    run_scrapy_spider(ShixinKuaichaSpider)
示例#10
0
def crawl_shixin_baidu():
    run_scrapy_spider(ShixinBaiduSpider)