def run(self):
start_mark = False
crawler = Crawler()
while self.flag:
try:
url = 'http://hw.venndata.cn/proxy?num={num}'.format(
num=self.num)
response, _ = crawler.crawl(url=url)
html = response.text
if html:
data = json.loads(html)['data']
proxy_list = data.split('|')
if len(proxy_list) > 500:
old_len = len(PROXY_LIST)
PROXY_LIST.extend(proxy_list)
PROXY_LIST[0:old_len] = []
if not start_mark:
log.critical("代理启动成功！获取代理%s" % len(proxy_list))
start_mark = True
except Exception as e:
log.error('生成 Proxy Failed' + str(e))
time.sleep(self.interval)
log.info('代理关闭')
return
def log_maven_output(lines):
for line in lines:
line = line.strip()
if line.startswith("[WARNING]"):
line = line[10:].rstrip()
if len(line) > 0:
log.warning("{!r}", line)
elif line.startswith("[ERROR]"):
line = line[8:].rstrip()
if len(line) > 0:
log.error("{!r}", line)
else:
pass
def log_maven_output(lines):
for line in lines:
line = line.strip()
if line.startswith("[WARNING]"):
line = line[10:].rstrip()
if len(line) > 0:
log.warning("{!r}", line)
elif line.startswith("[ERROR]"):
line = line[8:].rstrip()
if len(line) > 0:
log.error("{!r}", line)
else:
pass
def cleanup(self):
'''
Process cleanup.
'''
# Driftnet doesnt always clean up after itself, so here we will look to
# see if the process is running, and if it isnt, then cleanup the
# pidfile if it exists.
pidfile = '/var/run/driftnet.pid'
if os.path.exists(pidfile):
try:
os.kill(int(open(pidfile).read()), 0)
except OSError, e:
log.error('DRIFTNET: Cannot kill driftnet pidfile, still alive.')
else:
os.remove(pidfile)
def cleanup(self):
'''
Process cleanup.
'''
# Driftnet doesnt always clean up after itself, so here we will look to
# see if the process is running, and if it isnt, then cleanup the
# pidfile if it exists.
pidfile = '/var/run/driftnet.pid'
if os.path.exists(pidfile):
try:
os.kill(int(open(pidfile).read()), 0)
except OSError, e:
log.error(
'DRIFTNET: Cannot kill driftnet pidfile, still alive.')
else:
os.remove(pidfile)
def callService(url, params):
'''Main function to call cninfo services.
Args:
url: api url. String
params: request params. Dict
Returns:
if service return code is not 200, then return ''.
if call service success, then return response Json string.
'''
params['access_token'] = getAccessToken()
paramsbyte = bytes(urllib.parse.urlencode(params), 'utf8')
response = urllib.request.urlopen(config.base_url + url, paramsbyte)
respContentStr = response.read()
respContent = json.loads(respContentStr)
respCode = respContent['resultcode']
if respCode == 401 or respCode == 404 or respCode == 405:
log.debug('Token invalid. Updating it from service')
getAccessTokenFromService()
return callService(url, params)
elif respCode != 200:
log.error('Api调用出错：' + respContent['resultmsg'])
return ''
else:
return respContentStr
