def run(self): start_mark = False crawler = Crawler() while self.flag: try: url = 'http://hw.venndata.cn/proxy?num={num}'.format( num=self.num) response, _ = crawler.crawl(url=url) html = response.text if html: data = json.loads(html)['data'] proxy_list = data.split('|') if len(proxy_list) > 500: old_len = len(PROXY_LIST) PROXY_LIST.extend(proxy_list) PROXY_LIST[0:old_len] = [] if not start_mark: log.critical("代理启动成功!获取代理%s" % len(proxy_list)) start_mark = True except Exception as e: log.error('生成 Proxy Failed' + str(e)) time.sleep(self.interval) log.info('代理关闭') return
def log_maven_output(lines): for line in lines: line = line.strip() if line.startswith("[WARNING]"): line = line[10:].rstrip() if len(line) > 0: log.warning("{!r}", line) elif line.startswith("[ERROR]"): line = line[8:].rstrip() if len(line) > 0: log.error("{!r}", line) else: pass
def cleanup(self): ''' Process cleanup. ''' # Driftnet doesnt always clean up after itself, so here we will look to # see if the process is running, and if it isnt, then cleanup the # pidfile if it exists. pidfile = '/var/run/driftnet.pid' if os.path.exists(pidfile): try: os.kill(int(open(pidfile).read()), 0) except OSError, e: log.error('DRIFTNET: Cannot kill driftnet pidfile, still alive.') else: os.remove(pidfile)
def cleanup(self): ''' Process cleanup. ''' # Driftnet doesnt always clean up after itself, so here we will look to # see if the process is running, and if it isnt, then cleanup the # pidfile if it exists. pidfile = '/var/run/driftnet.pid' if os.path.exists(pidfile): try: os.kill(int(open(pidfile).read()), 0) except OSError, e: log.error( 'DRIFTNET: Cannot kill driftnet pidfile, still alive.') else: os.remove(pidfile)
def callService(url, params): '''Main function to call cninfo services. Args: url: api url. String params: request params. Dict Returns: if service return code is not 200, then return ''. if call service success, then return response Json string. ''' params['access_token'] = getAccessToken() paramsbyte = bytes(urllib.parse.urlencode(params), 'utf8') response = urllib.request.urlopen(config.base_url + url, paramsbyte) respContentStr = response.read() respContent = json.loads(respContentStr) respCode = respContent['resultcode'] if respCode == 401 or respCode == 404 or respCode == 405: log.debug('Token invalid. Updating it from service') getAccessTokenFromService() return callService(url, params) elif respCode != 200: log.error('Api调用出错:' + respContent['resultmsg']) return '' else: return respContentStr