class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
def __validProxy(self):
"""
验证代理
:return:
"""
while True:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
if validUsefulProxy(each_proxy):
# 成功计数器加1
self.db.inckey(each_proxy, 1)
self.log.debug(
'validProxy_b: {} validation pass'.format(each_proxy))
else:
# 失败计数器减一
self.db.inckey(each_proxy, -1)
# self.db.delete(each_proxy)
self.log.info(
'validProxy_b: {} validation fail'.format(each_proxy))
value = self.db.getvalue(each_proxy)
if value and int(value) < -5:
# 计数器小于-5删除该代理
self.db.delete(each_proxy)
self.log.info('validProxy_a running normal')
def main(self):
self.__validProxy()
class ProxyRefreshSchedule(ProxyManager):
"""
代理定时刷新
"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('refresh_schedule')
def validProxy(self):
"""
验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s start validProxy_a' % time.ctime())
exist_proxy = self.db.getAll()
while raw_proxy:
if validUsefulProxy(raw_proxy) and (raw_proxy not in exist_proxy):
self.db.changeTable(self.useful_proxy_queue)
self.db.put(raw_proxy)
self.log.info('validProxy_a: %s validation pass' % raw_proxy)
else:
self.log.debug('validProxy_a: %s validation fail' % raw_proxy)
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s validProxy_a complete' % time.ctime())
class NetEase(object):
def __init__(self):
"""
构造默认 header request session
"""
self.header = {
"Accept":
"*/*",
"Accept-Encoding":
"gzip,deflate,sdch",
"Accept-Language":
"zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4",
"Connection":
"keep-alive",
"Content-Type":
"application/x-www-form-urlencoded",
"Host":
"music.163.com",
"Referer":
"http://music.163.com",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
}
self.session = requests.session()
self.log = LogHandler('NeteaseApi')
def _raw_request(self, method, url, data=None):
"""
实际发起请求方法
:param method: POST | GET
:param url: url
:param data: 请求携带的数据
:return: response
"""
if method == "GET":
response = self.session.get(url,
params=data,
headers=self.header,
timeout=DEFAULT_TIMEOUT)
elif method == "POST":
response = self.session.post(url,
data=data,
headers=self.header,
timeout=DEFAULT_TIMEOUT)
return response
def _get_form_data(self, encrypt_data):
"""
获取加密后的 form data 参数
:param encrypt_data: 待加密的参数
:return: 加密后的参数 {"params":"", "encSecKey":""}
"""
key = netease.create_key(16)
return {
"params": netease.aes(netease.aes(encrypt_data, netease.NONCE),
key),
"encSecKey": netease.rsa(key, netease.PUBKEY, netease.MODULUS)
}
def request(self, method, path, data={}, default={"code": -1}):
"""
统一请求方法
:param method: POST | GET
:param path: 路径
:param data: 未加密的 data
:param default: 默认的 response
:return: response
"""
url = "{}{}".format(BASE_URL, path)
response = default
csrf_token = ""
data.update({"csrf_token": csrf_token})
params = self._get_form_data(json.dumps(data).encode('utf-8'))
try:
self.log.debug(
'[Netease api] url: {};\trequest data: {};\tparams: {}'.
format(url, data, params))
response = self._raw_request(method, url, params)
response = response.json()
self.log.debug('[Netease api] url: {};\tresponse data: {}'.format(
url, response))
except requests.exceptions.RequestException as e:
self.log.error('[Netease api] request error: {}'.format(e))
except ValueError as e:
self.log.error(
"[Netease api] request error; Path: {}, response: {}".format(
path, response.text[:200]))
finally:
return response
def songs_url(self, song_id):
"""
获取音乐的实际 url，外链
{ids: "[514235010]", level: "standard", encodeType: "aac", csrf_token: ""}
:param song_id: 音乐 id
:return: 带有外链的 json 串
"""
path = "/weapi/song/enhance/player/url/v1?csrf_token="
params = {
'ids': '[' + str(song_id) + ']',
'level': 'standard',
'encodeType': 'aac',
'csrf_token': ''
}
return self.request(POST, path, params)
def songs_lyric(self, song_id):
"""
获取音乐歌词
{id: "186453", lv: -1, tv: -1, csrf_token: ""}
:param song_id:
:return:
"""
path = "/weapi/song/lyric?csrf_token="
params = {'id': str(song_id), 'lv': -1, 'tv': -1, 'csrf_token': ''}
return self.request(POST, path, params)
def songs_search(self, keyword, offset=0, limit=30):
"""
搜索音乐
按照关键字搜索一般就用这个
{hlpretag: "<span class="s-fc7">", hlposttag: "</span>", s: "春夏秋冬 张国荣", type: "1", offset: "0", …}
:return:
"""
path = '/weapi/cloudsearch/get/web?csrf_token='
params = {
'csrf_token': '',
'hlposttag': '</span>',
'hlpretag': '<span class="s-fc7">',
'limit': str(limit),
'offset': str(offset),
's': str(keyword),
'total': 'true',
'type': '1'
}
return self.request(POST, path, params)
def songs_search_(self, song):
"""
搜索音乐，搜索框联动接口，不常用
{s: "春夏秋冬", limit: "8", csrf_token: ""}
:return:
"""
path = "/weapi/search/suggest/web?csrf_token="
params = {'s': str(song), 'limit': 8, 'csrf_token': ''}
return self.request(POST, path, params)
def songs_detail(self, song_id):
"""
获取歌曲详情
给定 song id
{id: "186453", c: "[{"id":"186453"}]", csrf_token: ""}
:param song_id: 必传参数，song id
:return: Song
"""
path = "/weapi/v3/song/detail?csrf_token="
params = {
'id': str(song_id),
'c': "[{'id': " + str(song_id) + "}]",
'csrf_token': ''
}
return self.request(POST, path, params)