import sys, re, random, os, threading, time, _thread
sys.path.append('../../lib/')
import kl_http, kl_db, kl_reg, kl_progress, kl_log
from urllib.parse import urlparse
log = kl_log.kl_log('bokedaquan')
regex = kl_reg
http = kl_http.kl_http()
mylock = _thread.allocate_lock() #线程锁
#mylock.acquire() #Get the lock
#mylock.release() #Release the lock.
progress = kl_progress.kl_progress('正在采集中')
progress.start()
#最大线程
maxthread = 10
threadnum = 0
#是否用代理
isproxy = False
http.setproxy('', '', '127.0.0.1:8087')
db = kl_db.mysql({
'host': 'localhost',
'user': '******',
'passwd': 'adminrootkl',
'db': 'bokedaquan',
'prefix': 'kl_',
'charset': 'utf8'
})
f = open('../proxy/proxy.txt', 'r')
s = f.read()
f.close()
import sys,re,random,os,threading,time,_thread
sys.path.append('../../lib/')
import kl_http,kl_db,kl_reg,kl_progress,kl_log
from urllib.parse import urlparse
log=kl_log.kl_log('bokedaquan')
regex=kl_reg
http=kl_http.kl_http()
mylock = _thread.allocate_lock()#线程锁
#mylock.acquire() #Get the lock
#mylock.release() #Release the lock.
progress=kl_progress.kl_progress('正在采集中')
progress.start()
#最大线程
maxthread=10
threadnum=0
#是否用代理
isproxy=False
http.setproxy('','','127.0.0.1:8087')
db=kl_db.mysql({
'host':'localhost',
'user':'******',
'passwd':'adminrootkl',
'db':'bokedaquan',
'prefix':'kl_',
'charset':'utf8'
})
f=open('../proxy/proxy.txt','r')
s=f.read()
f.close()
'ip':filterhtml(ip1),
'port':filterhtml(port1),
'niming':filterhtml(niming1),
'proxy_type':filterhtml(proxy_type1),
'proxy_area':filterhtml(proxy_area1),
'update_time':time.time(),
'area':i['area']
}
result=db.table('proxy').where(ma).count()
if result<=0:
print('添加新代理:%s:%s %s %s'%(ma['ip'],ma['port'],ma['niming'],ma['proxy_type']))
db.table('proxy').add(ma)
progress=kl_progress.kl_progress('')
progress.start()
progress.hide()
#测试代理是否可用
mylock = _thread.allocate_lock() #线程锁
#测试线程函数
def testProxy(i):
global curnum
#print('正在测试代理:%s:%s %s %s'%(i['ip'],i['port'],i['proxy_type'],i['proxy_area']))
# sys.stdout.write('正在测试代理:%s:%s ...'%(i['ip'],i['port'])+"\r")
# sys.stdout.flush()
progress.settext('正在测试代理:%s:%s'%(i['ip'],i['port']))
ht=kl_http.kl_http()
ht.setproxy('','','%s:%s'%(i['ip'],i['port']))
r=ht.geturl('http://proxy.59vip.cn')
mylock.acquire() #Get the lock
