def __init__(self, pool_size=15):
socket.setdefaulttimeout(20) #设置代理检验的超时时长
self.conn = sqlite3.connect('proxy.db', check_same_thread=False)
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.pool_size = pool_size
self.lock = threading.Lock()
self.raw_queue = Queue()
def __init__(self):
socket.setdefaulttimeout(30) #设置代理检验的超时时长
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.logger.p_log('开始获取代理...')
#类实例化时，打开数据库并清空数据
try:
self.conn.execute(
'create table if not exists proxy (host CHAR, port CHAR, proxy_type CHAR, user CHAR, password CHAR, checked BOOL, speed INT, catch_from CHAR, catch_time TIME)'
)
self.conn.execute('delete from proxy')
self.conn.commit()
except Exception, e:
self.logger.p_log('打开数据库时出错，程序退出...')
return False
class MyProxy():
conn = sqlite3.connect('proxy.db')
def __init__(self):
socket.setdefaulttimeout(30) #设置代理检验的超时时长
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.logger.p_log('开始获取代理...')
#类实例化时，打开数据库并清空数据
try:
self.conn.execute('create table if not exists proxy (host CHAR, port CHAR, proxy_type CHAR, user CHAR, password CHAR, checked BOOL, speed INT, catch_from CHAR, catch_time TIME)')
self.conn.execute('delete from proxy')
self.conn.commit()
except Exception, e:
self.logger.p_log('打开数据库时出错，程序退出...')
return False
def __init__(self, pool_size=15):
socket.setdefaulttimeout(20) #设置代理检验的超时时长
self.conn = sqlite3.connect('proxy.db', check_same_thread=False)
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.pool_size = pool_size
self.lock = threading.Lock()
self.raw_queue = Queue()
class VerifyProxy():
def __init__(self, pool_size=15):
socket.setdefaulttimeout(20) #设置代理检验的超时时长
self.conn = sqlite3.connect('proxy.db', check_same_thread=False)
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.pool_size = pool_size
self.lock = threading.Lock()
self.raw_queue = Queue()
def verify_proxy(self):
self.logger.p_log('开始检验代理...')
#从数据库中获取需要检验的代理列表
try:
proxies = self.conn.execute(
'select distinct host, port from proxy').fetchall()
self.logger.p_log('从数据库中获取到不重复的代理共 %d 个，准备逐个校验中' % len(proxies))
proxies = proxies[20:30]
except Exception, e:
self.logger.p_log('读取代理数据库错误，退出...')
return False
#生成线程池，大小为pool_size，缺省为10
for i in range(self.pool_size):
thread = threading.Thread(target=self.__verify_proxy)
thread.setDaemon(True)
thread.start()
#将获取到的待检验的代理放入序列proxy_queue中
for host, port in proxies:
self.raw_queue.put({'host': str(host), 'port': str(port)})
#将队列加入线程池，程序开始，直至队列为空
self.raw_queue.join()
self.__del__()
class VerifyProxy():
def __init__(self, pool_size=15):
socket.setdefaulttimeout(20) #设置代理检验的超时时长
self.conn = sqlite3.connect('proxy.db', check_same_thread=False)
self.logger = Logger('proxy.log') #用于保存log信息的文件
self.pool_size = pool_size
self.lock = threading.Lock()
self.raw_queue = Queue()
def verify_proxy(self):
self.logger.p_log('开始检验代理...')
#从数据库中获取需要检验的代理列表
try:
proxies = self.conn.execute('select distinct host, port from proxy').fetchall()
self.logger.p_log('从数据库中获取到不重复的代理共 %d 个，准备逐个校验中' % len(proxies))
proxies = proxies[20:30]
except Exception, e:
self.logger.p_log('读取代理数据库错误，退出...')
return False
#生成线程池，大小为pool_size，缺省为10
for i in range(self.pool_size):
thread = threading.Thread(target=self.__verify_proxy)
thread.setDaemon(True)
thread.start()
#将获取到的待检验的代理放入序列proxy_queue中
for host, port in proxies:
self.raw_queue.put({'host': str(host), 'port': str(port)})
#将队列加入线程池，程序开始，直至队列为空
self.raw_queue.join()
self.__del__()
