def crawl_baidu_shouji(n, proxies=True, debug=False):
"""
根据分类首页获得最大页码决定范围，暂时只适配百度url
:param n:
:param proxies:
:param debug:
:return:
"""
url = 'http://shouji.baidu.com/software/%s/' % n
while True:
html = get_page(url, proxies)
doc = pq(html)
try:
page_num = int(doc('.next').prev('li').children().text())
except ValueError as e:
print('获取信息失败: ', e)
continue
print('total page: ', page_num)
if debug:
page_num = 2
for pn in range(1, page_num + 1):
sub_url = 'http://shouji.baidu.com/software/%s/list_%s.html' % (n,
pn)
print()
print(
'=============================================================='
)
print('正在爬: ', sub_url)
html = get_page(sub_url, proxies, selector='p.down-btn > span')
yield html
break
def crawl_data5u(self):
start_url = 'http://www.data5u.com/free/gngn/index.shtml'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Cookie':
'JSESSIONID=47AA0C887112A2D83EE040405F837A86',
'Host':
'www.data5u.com',
'Referer':
'http://www.data5u.com/free/index.shtml',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
}
html = get_page(start_url, options=headers)
if html:
ip_address = re.compile(
'<span><li>(\d+\.\d+\.\d+\.\d+)</li>.*?<li class=\"port.*?>(\d+)</li>',
re.S)
re_ip_address = ip_address.findall(html)
for address, port in re_ip_address:
result = address + ':' + port
yield result.replace(' ', '')
def crawl_89ip(self):
start_url = 'http://www.89ip.cn/apijk/?&tqsl=1000&sxa=&sxb=&tta=&ports=&ktip=&cf=1'
html = get_page(start_url)
if html:
find_ips = re.compile('(\d+\.\d+\.\d+\.\d+:\d+)', re.S)
ip_ports = find_ips.findall(html)
for address_port in ip_ports:
yield address_port
def crawl_ip181(self):
start_url = 'http://www.ip181.com/'
html = get_page(start_url)
ip_address = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
# \s* 匹配空格，起到换行作用
re_ip_address = ip_address.findall(html)
for address, port in re_ip_address:
result = address + ':' + port
yield result.replace(' ', '')
def crawl_premproxy(self):
for i in ['China-01', 'China-02', 'China-03', 'China-04', 'Taiwan-01']:
start_url = 'https://premproxy.com/proxy-by-country/{}.htm'.format(
i)
html = get_page(start_url)
if html:
ip_address = re.compile('<td data-label="IP:port ">(.*?)</td>')
re_ip_address = ip_address.findall(html)
for address_port in re_ip_address:
yield address_port.replace(' ', '')
def crawl_ip3366(self):
for page in range(1, 4):
start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(
page)
html = get_page(start_url)
ip_address = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
# \s * 匹配空格，起到换行作用
re_ip_address = ip_address.findall(html)
for address, port in re_ip_address:
result = address + ':' + port
yield result.replace(' ', '')
def crawl_kxdaili(self):
for i in range(1, 11):
start_url = 'http://www.kxdaili.com/ipList/{}.html#ip'.format(i)
html = get_page(start_url)
ip_address = re.compile(
'<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
# \s* 匹配空格，起到换行作用
re_ip_address = ip_address.findall(html)
for address, port in re_ip_address:
result = address + ':' + port
yield result.replace(' ', '')
def crawl_kuaidaili(self):
for i in range(1, 4):
start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i)
html = get_page(start_url)
if html:
ip_address = re.compile('<td data-title="IP">(.*?)</td>')
re_ip_address = ip_address.findall(html)
port = re.compile('<td data-title="PORT">(.*?)</td>')
re_port = port.findall(html)
for address, port in zip(re_ip_address, re_port):
address_port = address + ':' + port
yield address_port.replace(' ', '')
def crawl_goubanjia(self):
"""
获取Goubanjia
:return: 代理
"""
start_url = 'http://www.goubanjia.com/free/gngn/index.shtml'
html = get_page(start_url)
if html:
doc = pq(html)
tds = doc('td.ip').items()
for td in tds:
td.find('p').remove()
yield td.text().replace(' ', '')
def crawl_proxy360(self):
"""
获取Proxy360
:return: 代理
"""
start_url = 'http://www.proxy360.cn/Region/China'
print('Crawling', start_url)
html = get_page(start_url)
if html:
doc = pq(html)
lines = doc('div[name="list_proxy_ip"]').items()
for line in lines:
ip = line.find('.tbBottomLine:nth-child(1)').text()
port = line.find('.tbBottomLine:nth-child(2)').text()
yield ':'.join([ip, port])
def crawl_iphai(self):
start_url = 'http://www.iphai.com/'
html = get_page(start_url)
if html:
find_tr = re.compile('<tr>(.*?)</tr>', re.S)
trs = find_tr.findall(html)
for s in range(1, len(trs)):
find_ip = re.compile('<td>\s+(\d+\.\d+\.\d+\.\d+)\s+</td>',
re.S)
re_ip_address = find_ip.findall(trs[s])
find_port = re.compile('<td>\s+(\d+)\s+</td>', re.S)
re_port = find_port.findall(trs[s])
for address, port in zip(re_ip_address, re_port):
address_port = address + ':' + port
yield address_port.replace(' ', '')
def crawl_ip3366(self):
for i in range(1, 4):
start_url = 'http://www.ip3366.net/?stype=1&page={}'.format(i)
html = get_page(start_url)
if html:
find_tr = re.compile('<tr>(.*?)</tr>', re.S)
trs = find_tr.findall(html)
for s in range(1, len(trs)):
find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>')
re_ip_address = find_ip.findall(trs[s])
find_port = re.compile('<td>(\d+)</td>')
re_port = find_port.findall(trs[s])
for address, port in zip(re_ip_address, re_port):
address_port = address + ':' + port
yield address_port.replace(' ', '')
def crawl_xroxy(self):
for i in ['CN', 'TW']:
start_url = 'http://www.xroxy.com/proxylist.php?country={}'.format(
i)
html = get_page(start_url)
if html:
ip_address1 = re.compile(
"title='View this Proxy details'>\s*(.*).*")
re_ip_address1 = ip_address1.findall(html)
ip_address2 = re.compile(
"title='Select proxies with port number .*'>(.*)</a>")
re_ip_address2 = ip_address2.findall(html)
for address, port in zip(re_ip_address1, re_ip_address2):
address_port = address + ':' + port
yield address_port.replace(' ', '')
def crawl_daili66(self, page_count=4):
"""
获取代理66
:param page_count: 页码
:return: 代理
"""
start_url = 'http://www.66ip.cn/{}.html'
urls = [start_url.format(page) for page in range(1, page_count + 1)]
for url in urls:
print('Crawling', url)
html = get_page(url)
if html:
doc = pq(html)
trs = doc('.containerbox table tr:gt(0)').items()
for tr in trs:
ip = tr.find('td:nth-child(1)').text()
port = tr.find('td:nth-child(2)').text()
yield ':'.join([ip, port])
def crawl_xicidaili(self):
for i in range(1, 3):
start_url = 'http://www.xicidaili.com/nn/{}'.format(i)
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Cookie':
'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRjYzc5MmM1MTBiMDMzYTUzNTZjNzA4NjBhNWRjZjliBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUp6S2tXT3g5a0FCT01ndzlmWWZqRVJNek1WanRuUDBCbTJUN21GMTBKd3M9BjsARg%3D%3D--2a69429cb2115c6a0cc9a86e0ebe2800c0d471b3',
'Host': 'www.xicidaili.com',
'Referer': 'http://www.xicidaili.com/nn/3',
'Upgrade-Insecure-Requests': '1',
}
html = get_page(start_url, options=headers)
if html:
find_trs = re.compile('<tr class.*?>(.*?)</tr>', re.S)
trs = find_trs.findall(html)
for tr in trs:
find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>')
re_ip_address = find_ip.findall(tr)
find_port = re.compile('<td>(\d+)</td>')
re_port = find_port.findall(tr)
for address, port in zip(re_ip_address, re_port):
address_port = address + ':' + port
yield address_port.replace(' ', '')
def crawl_chandashi_ios(keyword, proxies=True, debug=False):
"""
根据分类首页获得最大页码决定范围，暂时只适配百度url
:param n:
:param proxies:
:param debug:
:return:
"""
base_url = 'https://www.chandashi.com'
url = base_url + '/search/index.html?keyword=%s&type=store' % keyword
while True:
html = get_page(url, proxies, selector='#searchlist')
doc = pq(html)
try:
ios_url = base_url + doc(
'#searchlist > div:nth-child(1) > div > a').attr('href')
ios_html = get_page(ios_url, proxies)
print(
'=============================================================='
)
print('正在爬: ', ios_url)
ios_doc = pq(ios_html)
android_url = base_url + ios_doc(
'#pageTop > header > nav.navbar.nav-an.mobile-hide > div > div > ul > li:nth-child(2) > a'
).attr('href')
android_html = get_page(android_url, proxies)
yield ios_html, android_html
except Exception as e:
print('获取信息失败: ', e)
continue
#
# def parse_baidu_shouji(html):
# doc = pq(html)
# down_btn = doc('p.down-btn > span')
# if down_btn:
# print('本页有%s个应用' % down_btn.length)
# for d in down_btn.items():
# name = d.attr('data_name').strip()
# apk_name = d.attr('data_package').strip()
# if name and apk_name:
# data = {
# 'name': name,
# 'apk_name': apk_name
# }
# yield data
# else:
# print('无数据！')
#
#
# def main(n):
# """
# 所有环节串联
# :param n:
# :return:
# """
# htmls = crawl_baidu_shouji(n, proxies=settings.PROXY_MODE, debug=settings.DEBUG)
# for html in htmls:
# items = parse_baidu_shouji(html)
# try:
# mongo = APPNameMongoDBHandler(settings.MONGO_URL, settings.MONGO_DB, 'app_name')
# mongo.save_to_table('apk_name', items)
# except Exception as e:
# print('MongoDB错误:', list(items), '被忽略', e)
# print('%s页抓取完成！' % n)
#
#
# @timer
# def run():
# if settings.DEBUG:
# page_range = 501, 502
# else:
# page_range = settings.PAGE_RANGE
#
# pool = Pool()
# pool.map(main, [n for n in range(*page_range)])
#
#
# if __name__ == '__main__':
# run()