class MyProxy(): conn = sqlite3.connect('proxy.db') def __init__(self): socket.setdefaulttimeout(30) #设置代理检验的超时时长 self.logger = Logger('proxy.log') #用于保存log信息的文件 self.logger.p_log('开始获取代理...') #类实例化时,打开数据库并清空数据 try: self.conn.execute('create table if not exists proxy (host CHAR, port CHAR, proxy_type CHAR, user CHAR, password CHAR, checked BOOL, speed INT, catch_from CHAR, catch_time TIME)') self.conn.execute('delete from proxy') self.conn.commit() except Exception, e: self.logger.p_log('打开数据库时出错,程序退出...') return False
class MyProxy(): conn = sqlite3.connect('proxy.db') def __init__(self): socket.setdefaulttimeout(30) #设置代理检验的超时时长 self.logger = Logger('proxy.log') #用于保存log信息的文件 self.logger.p_log('开始获取代理...') #类实例化时,打开数据库并清空数据 try: self.conn.execute( 'create table if not exists proxy (host CHAR, port CHAR, proxy_type CHAR, user CHAR, password CHAR, checked BOOL, speed INT, catch_from CHAR, catch_time TIME)' ) self.conn.execute('delete from proxy') self.conn.commit() except Exception, e: self.logger.p_log('打开数据库时出错,程序退出...') return False
class VerifyProxy(): def __init__(self, pool_size=15): socket.setdefaulttimeout(20) #设置代理检验的超时时长 self.conn = sqlite3.connect('proxy.db', check_same_thread=False) self.logger = Logger('proxy.log') #用于保存log信息的文件 self.pool_size = pool_size self.lock = threading.Lock() self.raw_queue = Queue() def verify_proxy(self): self.logger.p_log('开始检验代理...') #从数据库中获取需要检验的代理列表 try: proxies = self.conn.execute( 'select distinct host, port from proxy').fetchall() self.logger.p_log('从数据库中获取到不重复的代理共 %d 个,准备逐个校验中' % len(proxies)) proxies = proxies[20:30] except Exception, e: self.logger.p_log('读取代理数据库错误,退出...') return False #生成线程池,大小为pool_size,缺省为10 for i in range(self.pool_size): thread = threading.Thread(target=self.__verify_proxy) thread.setDaemon(True) thread.start() #将获取到的待检验的代理放入序列proxy_queue中 for host, port in proxies: self.raw_queue.put({'host': str(host), 'port': str(port)}) #将队列加入线程池,程序开始,直至队列为空 self.raw_queue.join() self.__del__()
class VerifyProxy(): def __init__(self, pool_size=15): socket.setdefaulttimeout(20) #设置代理检验的超时时长 self.conn = sqlite3.connect('proxy.db', check_same_thread=False) self.logger = Logger('proxy.log') #用于保存log信息的文件 self.pool_size = pool_size self.lock = threading.Lock() self.raw_queue = Queue() def verify_proxy(self): self.logger.p_log('开始检验代理...') #从数据库中获取需要检验的代理列表 try: proxies = self.conn.execute('select distinct host, port from proxy').fetchall() self.logger.p_log('从数据库中获取到不重复的代理共 %d 个,准备逐个校验中' % len(proxies)) proxies = proxies[20:30] except Exception, e: self.logger.p_log('读取代理数据库错误,退出...') return False #生成线程池,大小为pool_size,缺省为10 for i in range(self.pool_size): thread = threading.Thread(target=self.__verify_proxy) thread.setDaemon(True) thread.start() #将获取到的待检验的代理放入序列proxy_queue中 for host, port in proxies: self.raw_queue.put({'host': str(host), 'port': str(port)}) #将队列加入线程池,程序开始,直至队列为空 self.raw_queue.join() self.__del__()