def _add_clear_proxys(self): clear_proxys = [] if SCAN_FREE_DAILI is False: # clear_proxys.append("benxiaohai:[email protected]:62011") # clear_proxys.append("benxiaohai:[email protected]:62012") # clear_proxys.append("benxiaohai:[email protected]:62013") # clear_proxys.append("benxiaohai:[email protected]:62014") # clear_proxys.append("benxiaohai:[email protected]:62015") # clear_proxys.append("benxiaohai:[email protected]:62016") # clear_proxys.append("benxiaohai:[email protected]:62017") # clear_proxys.append("benxiaohai:[email protected]:62018") # clear_proxys.append("benxiaohai:[email protected]:62019") # clear_proxys.append("benxiaohai:[email protected]:62020") clear_proxys.extend(PROXIES) global proxy_ip_db_mutex if proxy_ip_db_mutex.acquire(): for ip in clear_proxys: new_proxy_ip = ProxyIp() new_proxy_ip.ip = ip try: new_proxy_ip.save() self.add_ip(ip) except: pass proxy_ip_db_mutex.release()
def run(self): while not self.stop_me: verify_threads = [] ProxyIp.drop_collection() # @UndefinedVariable proxy_logger.info("new proxy ips updating:") start = time.time() try: if SCAN_FREE_DAILI: IPPool = getIPs() for thread_id in range(0, VERIFY_PROXY_IP_NUMBER): ips = [] index = thread_id while index < len(IPPool): ips.append(IPPool[index]) index += VERIFY_PROXY_IP_NUMBER verify_threads.append(ProxyVerifier("verify " + str(thread_id), ips, self)) print "verify thread %d, ip_list'size is %d" % (thread_id, len(ips)) self.mid_ip_list = [] self._add_clear_proxys() if SCAN_FREE_DAILI: for verify_thread in verify_threads: verify_thread.start() for verify_thread in verify_threads: verify_thread.join() proxy_logger.info("all verify threads end!") self.ip_list = self.mid_ip_list end = time.time() proxy_logger.info("end proxy ips updating: {total time: " + str(end - start) + ", nowtime: " + str(datetime.datetime.now()) + ", current ip_list siz: " + str(len(self.ip_list)) + "}") proxy_logger.info("current available ips: ") for ip in self.ip_list: proxy_logger.info(ip) except : s = traceback.format_exc() print s pass self._sleep_for_next_update()
def run(self): for ip in self.ip_list: validity = False try: validity = self._verify_ip(ip) except: pass if validity: self.proxy_manager.add_ip(ip) global proxy_ip_db_mutex if proxy_ip_db_mutex.acquire(): new_proxy_ip = ProxyIp() new_proxy_ip.ip = ip try: new_proxy_ip.save() except: pass finally: proxy_ip_db_mutex.release()