def crawl_proxy_web(self):
spiders = [
XiciSpider,
# Data5uSpider,
KuaidailiSpider,
]
spiders = random.choice(spiders)
models = spiders.get_proxies()
filtered_models = requestEnginer.filter_unavailable_proxy(models)
for each in filtered_models:
self.dbmanager.insert_proxy_table(each)
def crawl_proxyWeb(self):
spiders = [
# data5uSpider,
# kuaidailiSpider,
# ip181Spider,
xiciSpider, # 目前只用西刺代理
]
for spider in spiders:
models = spider.getProxies()
filtered_models = requestEnginer.filter_unavailable_proxy(models)
for each in filtered_models:
self.dbmanager.insert_proxy_table(each)
def crawl_proxy_web(self):
spiders = [
XiciSpider,
Data5uSpider,
KuaidailiSpider,
Ip181Spider,
]
for spider in spiders:
models = spider.get_proxies()
filtered_models = requestEnginer.filter_unavailable_proxy(models)
for each in filtered_models:
self.dbmanager.insert_proxy_table(each)