def freeProxyFifth(): """ guobanjia http://www.goubanjia.com/ :return: """ url = "http://www.goubanjia.com/" tree = EasyHttp.getHtmlTree(url) if tree is None: Log.w('http://www.goubanjia.com无效') return [] proxy_list = tree.xpath('//td[@class="ip"]') # 此网站有隐藏的数字干扰,或抓取到多余的数字或.符号 # 需要过滤掉<p style="display:none;">的内容 xpath_str = """.//*[not(contains(@style, 'display: none')) and not(contains(@style, 'display:none')) and not(contains(@class, 'port')) ]/text() """ for each_proxy in proxy_list: try: # :符号裸放在td下,其他放在div span p中,先分割找出ip,再找port ip_addr = ''.join(each_proxy.xpath(xpath_str)) port = each_proxy.xpath( ".//span[contains(@class, 'port')]/text()")[0] yield '{}:{}'.format(ip_addr, port) except Exception as e: pass
def freeProxySeventh(): """ 快代理 https://www.kuaidaili.com """ url_list = [ 'https://www.kuaidaili.com/free/inha/{page}/', 'https://www.kuaidaili.com/free/intr/{page}/' ] for url in url_list: for page in range(1, 2): page_url = url.format(page=page) tree = EasyHttp.getHtmlTree(page_url) if tree is None: Log.w('http://www.kuaidaili.com无效') return [] proxy_list = tree.xpath('.//table//tr') for tr in proxy_list[1:]: yield ':'.join(tr.xpath('./td/text()')[0:2])
def freeProxyTwelve(page_count=2): """ guobanjia http://ip.jiangxianli.com/?page= 免费代理库 超多量 :return: """ for i in range(1, page_count + 1): url = 'http://ip.jiangxianli.com/?page={}'.format(i) html_tree = EasyHttp.getHtmlTree(url) if not html_tree: Log.w('http://ip.jiangxianli.com无效') return [] tr_list = html_tree.xpath("/html/body/div[1]/div/div[1]/div[2]/table/tbody/tr") if len(tr_list) == 0: continue for tr in tr_list: yield tr.xpath("./td[2]/text()")[0] + ":" + tr.xpath("./td[3]/text()")[0]
def freeProxySecond(area=33, page=1): """ 代理66 http://www.66ip.cn/ :param area: 抓取代理页数,page=1北京代理页,page=2上海代理页...... :param page: 翻页 :return: """ area = 33 if area > 33 else area for area_index in range(1, area + 1): for i in range(1, page + 1): url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i) html_tree = EasyHttp.getHtmlTree(url) if not html_tree: Log.w('http://www.66ip.cn无效') return [] tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]") if len(tr_list) == 0: continue for tr in tr_list: yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0] break
def freeProxyFourth(page_count=2): """ 西刺代理 http://www.xicidaili.com :return: """ url_list = [ 'http://www.xicidaili.com/nn/', # 高匿 'http://www.xicidaili.com/nt/', # 透明 ] for each_url in url_list: for i in range(1, page_count + 1): page_url = each_url + str(i) tree = EasyHttp.getHtmlTree(page_url) if not tree: Log.w('http://www.xicidaili.com无效') return [] proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]') for proxy in proxy_list: try: yield ':'.join(proxy.xpath('./td/text()')[0:2]) except Exception as e: pass