def crawl_company_info(): spider_list = [ List51jobSpider, List51SoleSpider, List56YeSpider, List58Spider, List8671Spider, List88152Spider, List99114Spider, ListCnlinfo88Spider, ListCnlistSpider, ListGanjiSpider, ListHuangye88Spider, ListMakepoloSpider, ListQincaiSpider, ListQiyeSpider, ListQy6Spider, ListSoudhSpider, ListYnshangjiSpider, ListYouboySpider, ListZhaopinSpider, ListZhaoshang100Spider, DetailTianyanchaSpider, # 会被封 Detail58Spider, DetailSzmqsSpider, ] run_scrapy_spider(spider_list)
def crawl_new_proxy(): spider_list = [ # GoubanjiaSpider, KuaidailiSpider, XicidailiSpider, # NianshaoSpider, # 包含国外IP # Ip181Spider, # 包含国外IP # Ip3366Spider, # 包含国外IP ] run_scrapy_spider(spider_list)
def crawl_mobile_info(): run_scrapy_spider([ CnmoSpider, ])
def crawl_zhixing_dlm(): run_scrapy_spider(ShixinDLMSpider)
def crawl_wenshu_info(): run_scrapy_spider(WenshuPcSpider)
def crawl_zhixing_court(): run_scrapy_spider(ZhixingCourtSpider)
def crawl_shixin_p2p(): run_scrapy_spider([ ShixinKaikaidaiSpider, ShixinMy089Spider, ])
def crawl_shixin_other(): run_scrapy_spider([ ShixinCourtSpider, ShixinmingdanSpider, ShixinDLMSpider, ])
def crawl_shixin_kuaicha(): run_scrapy_spider(ShixinKuaichaSpider)
def crawl_shixin_baidu(): run_scrapy_spider(ShixinBaiduSpider)