def _get_tags(self, article_info): ''' 获取tags :return: ''' tmp_tags = list_duplicate_remove( [str(item.get('name', '')) for item in article_info.get('noteInfo', {}).get('relatedTags', [])]) # self.my_lg.info(str(tmp_tags)) # list先转str, 去掉敏感字眼, 再转list, 并去除''元素, 得到最后list tmp_tags = delete_list_null_str(self.wash_sensitive_info('|'.join(tmp_tags)).split('|')) tags = [{ # tags可以为空list! 'keyword': item, } for item in tmp_tags] return tags
def _get_66_ip_list(): ''' 先获取66高匿名ip :return: ''' global a_66_ip headers = { 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Referer': 'http://www.66ip.cn/nm.html', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', } params = ( ('getnum', ''), ('isp', '0'), ('anonymoustype', '3'), ('start', ''), ('ports', ''), ('export', ''), ('ipaddress', ''), ('area', '0'), ('proxytype', '2'), ('api', '66ip'), ) response = requests.get('http://www.66ip.cn/nmtq.php', headers=headers, params=params, cookies=None) body = Requests._wash_html(response.content.decode('gbk')) try: part = re.compile(r'</script>(.*)</div>').findall(body)[0] except IndexError: part = '' part = re.compile('<script>.*?</script>|</div>.*</div>').sub('', part) # print(part) ip_list = delete_list_null_str(part.split('<br />')) # print(ip_list) a_66_ip = ip_list if ip_list != [] else [] return ip_list
def _get_66_ip_list(): ''' 先获取66高匿名ip :return: ''' global ori_ip_list params = ( ('getnum', ''), ('isp', '0'), ('anonymoustype', '3'), ('start', ''), ('ports', ''), ('export', ''), ('ipaddress', ''), ('area', '0'), ('proxytype', '2'), ('api', '66ip'), ) with session() as s: try: response = s.get('http://www.66ip.cn/nmtq.php', headers=_get_base_headers(), params=params, cookies=None) except Exception: return [] body = Requests._wash_html(response.content.decode('gbk')) try: part = re.compile(r'</script>(.*)</div>').findall(body)[0] except IndexError: part = '' part = re.compile('<script>.*?</script>|</div>.*</div>').sub('', part) # print(part) ip_list = delete_list_null_str(part.split('<br />')) # print(ip_list) ori_ip_list = ip_list if ip_list != [] else [] return ip_list
def get_start_up_ip_list(url): ''' 初始抓取时调用 :param url: :return: ''' with session() as s: body = s.get(url, headers=_get_base_headers()).text if body == '': return [] tmp_ip_list = delete_list_null_str(body.split('\r\n')) ip_list = [] for item in tmp_ip_list: try: tmp = re.compile('\d+\.\d+\.\d+\.\d+:\d+').findall(item)[0] ip_list.append(tmp) except IndexError: continue return ip_list