def _add_clear_proxys(self):
clear_proxys = []
if SCAN_FREE_DAILI is False:
# clear_proxys.append("benxiaohai:[email protected]:62011")
# clear_proxys.append("benxiaohai:[email protected]:62012")
# clear_proxys.append("benxiaohai:[email protected]:62013")
# clear_proxys.append("benxiaohai:[email protected]:62014")
# clear_proxys.append("benxiaohai:[email protected]:62015")
# clear_proxys.append("benxiaohai:[email protected]:62016")
# clear_proxys.append("benxiaohai:[email protected]:62017")
# clear_proxys.append("benxiaohai:[email protected]:62018")
# clear_proxys.append("benxiaohai:[email protected]:62019")
# clear_proxys.append("benxiaohai:[email protected]:62020")
clear_proxys.extend(PROXIES)
global proxy_ip_db_mutex
if proxy_ip_db_mutex.acquire():
for ip in clear_proxys:
new_proxy_ip = ProxyIp()
new_proxy_ip.ip = ip
try:
new_proxy_ip.save()
self.add_ip(ip)
except:
pass
proxy_ip_db_mutex.release()
def run(self):
while not self.stop_me:
verify_threads = []
ProxyIp.drop_collection() # @UndefinedVariable
proxy_logger.info("new proxy ips updating:")
start = time.time()
try:
if SCAN_FREE_DAILI:
IPPool = getIPs()
for thread_id in range(0, VERIFY_PROXY_IP_NUMBER):
ips = []
index = thread_id
while index < len(IPPool):
ips.append(IPPool[index])
index += VERIFY_PROXY_IP_NUMBER
verify_threads.append(ProxyVerifier("verify " + str(thread_id), ips, self))
print "verify thread %d, ip_list'size is %d" % (thread_id, len(ips))
self.mid_ip_list = []
self._add_clear_proxys()
if SCAN_FREE_DAILI:
for verify_thread in verify_threads:
verify_thread.start()
for verify_thread in verify_threads:
verify_thread.join()
proxy_logger.info("all verify threads end!")
self.ip_list = self.mid_ip_list
end = time.time()
proxy_logger.info("end proxy ips updating: {total time: " + str(end - start) + ", nowtime: " + str(datetime.datetime.now()) + ", current ip_list siz: " + str(len(self.ip_list)) + "}")
proxy_logger.info("current available ips: ")
for ip in self.ip_list:
proxy_logger.info(ip)
except :
s = traceback.format_exc()
print s
pass
self._sleep_for_next_update()
def run(self):
for ip in self.ip_list:
validity = False
try:
validity = self._verify_ip(ip)
except:
pass
if validity:
self.proxy_manager.add_ip(ip)
global proxy_ip_db_mutex
if proxy_ip_db_mutex.acquire():
new_proxy_ip = ProxyIp()
new_proxy_ip.ip = ip
try:
new_proxy_ip.save()
except:
pass
finally:
proxy_ip_db_mutex.release()
