Esempi in Python per proxy_lists

Esempio n. 1

File: middlewares.py Progetto: XDDDD/apkcrawl

# -*- encoding=utf-8
import random
from apkcrawl import settings
from apkcrawl.utils import proxy_lists , random_proxy

proxies = proxy_lists( settings.PROXY_LIST_FILE )
 
class ProxyMiddleware(object):

    def process_request(self, request, spider):
        request.meta['proxy'] = random_proxy( proxies )[1]

class EmptyProxyMiddleware(object):

    def process_request(self, request, spider):
        pass

Esempio n. 2

 def test_proxy_lists(self):
     """test_proxy_lists 测试代理列表"""
     proxies = proxy_lists( settings.PROXY_LIST_FILE )
     self.assertTrue( len(proxies) != 0)

Esempio n. 3

File: middlewares.py Progetto: amumu/apkcrawl

# -*- encoding=utf-8
import random
from apkcrawl import settings
from apkcrawl.utils import proxy_lists, random_proxy

proxies = proxy_lists(settings.PROXY_LIST_FILE)


class ProxyMiddleware(object):
    def process_request(self, request, spider):
        request.meta['proxy'] = random_proxy(proxies)[1]


class EmptyProxyMiddleware(object):
    def process_request(self, request, spider):
        pass