def loc_proxy_ipv4(proxies):
test_res = requests_dora.try_best_2_get("http://lumtest.com/myip.json", max_times=2, proxies=proxies, timeout=10)
if test_res is not None and test_res.status_code == 200:
return(test_res.text)
else:
logging.warning("failed to connect to geolocation url ...")
return None
def get_singer_list_page(area_id, page_ind):
'''
:param area_id: {'全部': -100, '内地': 200, '港台': 2, '欧美': 5, '日本': 4, '韩国': 3, '其他': 6}
:param page_ind:
:return:
'''
url = "https://u.y.qq.com/cgi-bin/musicu.fcg"
header = {
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"cookie":
"RK=7dNm4/X + Yj; tvfe_boss_uuid=bf00ee54e9081ab4; pgv_pvi=8772238336; pac_uid=1_857193777; pgv_pvid=6457341280; o_cookie=857193777; ptcz=c761e59c8c8d6bd5198866d02a5cb7313af1af468006c455d6c2b5d26201d42e; pgv_si=s10759168; _qpsvr_localtk=0.08285763449905015; ptisp=ctc; luin=o0857193777; lskey=00010000228dd1371b945c68ecfd3b71d3071425024a7a8a2a23e3ffcb5b9904c9f7088d2ea8c01539ffed92; pt2gguin=o0857193777; uin=o0857193777; skey=@Kydi7w0EI; p_uin=o0857193777; p_skey=HjsE9sEjznJfXk*9KFEeW4VZr6i3*tlXZ2nuzEw8kCg_; pt4_token=c-p6sv3JEboA51cSQ3ABqxM8O80Jct3jYYkgy-aEQuE_; p_luin=o0857193777; p_lskey=000400008f9c296cd10c03a5173d22a184aad124d791568e90e4198beb8ad699a4d02fbfc059f71ab3d8758c; ts_last=y.qq.com/portal/playlist.html; ts_refer=ui.ptlogin2.qq.com/cgi-bin/login; ts_uid=3392060960",
"referer": "https://y.qq.com/portal/singer_list.html"
}
paramter = {
"g_tk":
"5381",
"callback":
"getUCGI9688380858412697",
"jsonpCallback":
"getUCGI9688380858412697",
"loginUin":
"0",
"hostUin":
"0",
"format":
"jsonp",
"inCharset":
"utf8",
"outCharset":
"utf-8",
"notice":
"0",
"platform":
"yqq",
"needNewCode":
"0",
"data":
'{"comm":{"ct":24,"cv":10000},"singerList":{"module":"Music.SingerListServer","method":"get_singer_list","param":{"area":%d,"sex":-100,"genre":-100,"index":-100,"sin":%d,"cur_page":%d}}}'
% (area_id, (page_ind - 1) * 80, page_ind)
}
html_text = requests_dora.try_best_2_get(url=url,
params=paramter,
headers=header).text
se = re.search("getUCGI9688380858412697\((.*)\)", html_text)
json_str = None
if se:
json_str = se.group(1)
data = json.loads(json_str)["singerList"]["data"]
singerlist = data["singerlist"]
try:
area_ids = data["tags"]["area"]
except Exception as e:
area_ids = None
singer_num_total = data["total"]
return singerlist, singer_num_total, area_ids
def get_TKK(get_proxies_fun=None):
'''
to get a value for calculating the token for constructing translating request
:return: TKK
'''
url = "https://translate.google.cn"
logging.warning("start getting tkk...")
res = requests_dora.try_best_2_get(url, get_proxies_fun=get_proxies_fun)
TKK = re.search("tkk:'(.*?)'", res.text).group(1)
return TKK
def get_data5u_proxies(api=None):
url = "http://api.ip.data5u.com/dynamic/get.html?order=53b3de376027aa3f699dc335d2bc0674&sep=3"
if api is not None:
url = api
res = requests_dora.try_best_2_get(url)
proxy = res.text.strip()
if not re.match("\d+\.\d+\.\d+\.\d+:\d+", proxy):
logging.warning("the proxy expired...")
raise Exception
return get_proxies(proxy)
def loc_proxy(proxies):
res = requests_dora.try_best_2_get("https://proxy6.net/en/myip", max_times=2, proxies=proxies, timeout=10)
soup = BeautifulSoup(res.text, "lxml")
# ip = soup.select_one("div.block-head > h1").get_text()
loc_div = soup.select_one("div.myip-row > div")
line_list = loc_div.select("dl")
loc_info = {}
for line in line_list:
key = line.select_one("dt").get_text().strip().lower()
val = line.select_one("dd").get_text().strip()
loc_info[key] = val
return loc_info
def get_song_list(dissid):
url = "https://c.y.qq.com/qzone/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg"
header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"cookie": "RK=7dNm4/X + Yj; tvfe_boss_uuid=bf00ee54e9081ab4; pgv_pvi=8772238336; pac_uid=1_857193777; pgv_pvid=6457341280; o_cookie=857193777; ptcz=c761e59c8c8d6bd5198866d02a5cb7313af1af468006c455d6c2b5d26201d42e; pgv_si=s10759168; _qpsvr_localtk=0.08285763449905015; ptisp=ctc; luin=o0857193777; lskey=00010000228dd1371b945c68ecfd3b71d3071425024a7a8a2a23e3ffcb5b9904c9f7088d2ea8c01539ffed92; pt2gguin=o0857193777; uin=o0857193777; skey=@Kydi7w0EI; p_uin=o0857193777; p_skey=HjsE9sEjznJfXk*9KFEeW4VZr6i3*tlXZ2nuzEw8kCg_; pt4_token=c-p6sv3JEboA51cSQ3ABqxM8O80Jct3jYYkgy-aEQuE_; p_luin=o0857193777; p_lskey=000400008f9c296cd10c03a5173d22a184aad124d791568e90e4198beb8ad699a4d02fbfc059f71ab3d8758c; ts_last=y.qq.com/portal/playlist.html; ts_refer=ui.ptlogin2.qq.com/cgi-bin/login; ts_uid=3392060960",
"referer": "https://y.qq.com/n/yqq/playlist/{}.html".format(dissid)
}
paramters = {
"type": "1",
"json": "1",
"utf8": "1",
"onlysong": "0",
"disstid": dissid,
"format": "jsonp",
"g_tk": "1089387893",
"jsonpCallback": "playlistinfoCallback",
"loginUin": "857193777",
"hostUin": "0",
"inCharset": "utf8",
"outCharset": "utf-8",
"notice": 0,
"platform": "yqq",
"needNewCode": 0
}
html_text = requests_dora.try_best_2_get(url=url, params=paramters, headers=header).text
cdlist = json.loads(html_text.lstrip("playlistinfoCallback(").rstrip(")"))["cdlist"]
if len(cdlist) >= 1:
cdlist = cdlist[0]
song_list = []
tags = ", ".join([i["name"] for i in cdlist["tags"]])
for item in cdlist["songlist"]:
song = {}
# if "size128" in item:
# song["size128"] = item["size128"]
if "songmid" in item:
song["songmid"] = item["songmid"]
else:
continue
if "songid" in item:
song["songid"] = item["songid"]
else:
continue
song["albumname"] = item["albumname"]
song["songname"] = item["songname"]
song["singer"] = ", ".join([i["name"] for i in item["singer"]])
song["tags"] = tags
song_list.append(song)
return song_list
def crawl_song_list_page(singer_mid, begin):
url = "https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg"
header = {
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"cookie":
"RK=7dNm4/X + Yj; tvfe_boss_uuid=bf00ee54e9081ab4; pgv_pvi=8772238336; pac_uid=1_857193777; pgv_pvid=6457341280; o_cookie=857193777; ptcz=c761e59c8c8d6bd5198866d02a5cb7313af1af468006c455d6c2b5d26201d42e; pgv_si=s10759168; _qpsvr_localtk=0.08285763449905015; ptisp=ctc; luin=o0857193777; lskey=00010000228dd1371b945c68ecfd3b71d3071425024a7a8a2a23e3ffcb5b9904c9f7088d2ea8c01539ffed92; pt2gguin=o0857193777; uin=o0857193777; skey=@Kydi7w0EI; p_uin=o0857193777; p_skey=HjsE9sEjznJfXk*9KFEeW4VZr6i3*tlXZ2nuzEw8kCg_; pt4_token=c-p6sv3JEboA51cSQ3ABqxM8O80Jct3jYYkgy-aEQuE_; p_luin=o0857193777; p_lskey=000400008f9c296cd10c03a5173d22a184aad124d791568e90e4198beb8ad699a4d02fbfc059f71ab3d8758c; ts_last=y.qq.com/portal/playlist.html; ts_refer=ui.ptlogin2.qq.com/cgi-bin/login; ts_uid=3392060960",
"referer": "https://y.qq.com/n/yqq/singer/{}.html".format(singer_mid)
}
paramter = {
"g_tk": "5381",
"jsonpCallback": "MusicJsonCallbacksinger_track",
"loginUin": "0",
"hostUin": "0",
"format": "jsonp",
"inCharset": "utf8",
"outCharset": "utf-8",
"notice": "0",
"platform": "yqq",
"needNewCode": "0",
"singermid": singer_mid,
"order": "listen",
"begin": begin,
"num": "30",
"songstatus": "1",
}
html_text = requests_dora.try_best_2_get(url=url,
params=paramter,
headers=header).text
json_str = html_text.lstrip(" MusicJsonCallbacksinger_track(").rstrip(
")").strip()
data = json.loads(json_str)["data"]
song_list = data["list"]
total_num = data["total"]
song_list_new = []
for song in song_list:
song = song["musicData"]
song_new = {}
song_new["albumid"] = song["albumid"]
song_new["albummid"] = song["albummid"]
song_new["albumname"] = song["albumname"]
song_new["songid"] = song["songid"]
song_new["songmid"] = song["songmid"]
song_new["songname"] = song["songname"]
song_list_new.append(song_new)
return song_list_new, int(total_num)
def google_search(queryStr, get_proxies_fun=None, page=1):
url = 'https://www.google.com/search?biw=1920&safe=active&hl=en&q=%s&oq=%s&start=%d' % (
queryStr, queryStr, (page - 1) * 10)
response = requests_dora.try_best_2_get(
url,
headers=requests_dora.get_default_headers(),
invoked_by="google_search",
get_proxies_fun=get_proxies_fun,
timeout=60)
status = response.status_code
if status == 200:
html = response.text
else:
print("status: {}, try again....".format(status))
random.seed(time.time())
time.sleep(3 + 5 * random.random())
return google_search(queryStr, page, get_proxies_fun)
return html
def get_album_list(sin, ein):
url = "https://c.y.qq.com/splcloud/fcgi-bin/fcg_get_diss_by_tag.fcg"
header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"cookie": "RK=7dNm4/X + Yj; tvfe_boss_uuid=bf00ee54e9081ab4; pgv_pvi=8772238336; pac_uid=1_857193777; pgv_pvid=6457341280; o_cookie=80; ptcz=c761e59c8c8d6bd5198866d02a5cb7313af1af468006c455d6c2b5d26201d42e; pgv_si=s10759168; _qpsvr_localtk=0.08285763449905015; ptisp=ctc; luin=o0857193777; lskey=00010000228dd1371b945c68ecfd3b71d3071425024a7a8a2a23e3ffcb5b9904c9f7088d2ea8c01539ffed92; pt2gguin=o0857193777; uin=o0857193777; skey=@Kydi7w0EI; p_uin=o0857193777; p_skey=HjsE9sEjznJfXk*9KFEeW4VZr6i3*tlXZ2nuzEw8kCg_; pt4_token=c-p6sv3JEboA51cSQ3ABqxM8O80Jct3jYYkgy-aEQuE_; p_luin=o0857193777; p_lskey=000400008f9c296cd10c03a5173d22a184aad124d791568e90e4198beb8ad699a4d02fbfc059f71ab3d8758c; ts_last=y.qq.com/portal/playlist.html; ts_refer=ui.ptlogin2.qq.com/cgi-bin/login; ts_uid=3392060960",
"referer": "https://y.qq.com/portal/playlist.html"
}
paramter = {
"g_tk": "1089387893",
"jsonpCallback": "getPlaylist",
"loginUin": "0",
"hostUin": "0",
"format": "jsonp",
"inCharset": "utf8",
"outCharset": "utf-8",
"notice": "0",
"platform": "yqq",
"needNewCode": "0",
"categoryId": "10000000",
"sortId": "5",
"sin": sin, # 开始结点
"ein": ein # 结束结点，用于翻页
}
html_text = requests_dora.try_best_2_get(url=url, params=paramter, headers=header).text
res = json.loads(html_text.lstrip("getPlaylist(").rstrip(")"))["data"]["list"]
album_list = []
for t_item in res:
album = {}
ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")#用于去掉非法字符
album["createtime"] = t_item["createtime"]
album["creator_qq"] = t_item["creator"]["qq"]
album["creator_name"] = t_item["creator"]["name"]
album["creator_name"] = ILLEGAL_CHARACTERS_RE.sub(r"", album["creator_name"])
album["creator_isVip"] = t_item["creator"]["isVip"]
album["dissid"] = t_item["dissid"] #提取歌单id，用于后续提取歌曲id
album["dissname"] = t_item["dissname"] #歌单名称
album["dissname"] = ILLEGAL_CHARACTERS_RE.sub(r"", album["dissname"])
album["listennum"] = t_item["listennum"] #播放量
album_list.append(album)
return album_list
def get_detail(song):
songid = song["songid"]
songmid = song["songmid"]
url = "https://c.y.qq.com/v8/fcg-bin/fcg_play_single_song.fcg"
header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"referer": "https://y.qq.com/n/yqq/song/{}.html".format(songid)
}
paramters = {
"songmid": songmid,
"tpl": "yqq_song_detail",
"format": "jsonp",
"callback": "getOneSongInfoCallback",
"g_tk": "1134533366",
"jsonpCallback": "getOneSongInfoCallback",
"loginUin": "0",
"hostUin": "0",
"inCharset": "utf8",
"outCharset": "utf-8",
"notice": 0,
"platform": "yqq",
"needNewCode": 0
}
html_text = requests_dora.try_best_2_get(url=url, params=paramters, headers=header, verify=True).text
detail = json.loads(html_text.lstrip("getOneSongInfoCallback(").rstrip(")"))["data"]
song = {}
if len(detail) > 0:
detail = detail[0]
song["subtitle"] = detail["subtitle"]
song["title"] = detail["title"]
song["time_public"] = detail["time_public"]
try:
song["url"] = json.loads(html_text.lstrip("getOneSongInfoCallback(").rstrip(")"))["url"][str(songid)]
except:
song["url"] = ""
return song
def get_lyric(song):
songid = song["songid"]
songmid = song["songmid"]
url = "https://c.y.qq.com/lyric/fcgi-bin/fcg_query_lyric.fcg"
header = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"referer": "https://y.qq.com/n/yqq/song/{}.html".format(songmid)
}
paramters = {
"nobase64": 1,
"musicid": songid,
"callback": "jsonp1",
"g_tk": "1134533366",
"jsonpCallback": "jsonp1",
"loginUin": "0",
"hostUin": "0",
"format": "jsonp",
"inCharset": "utf8",
"outCharset": "utf-8",
"notice": "0",
"platform": "yqq",
"needNewCode": "0"
}
html_text = requests_dora.try_best_2_get(url=url, params=paramters, headers=header).text
res = json.loads(html_text.lstrip("jsonp1(").rstrip(")"))
# if "lyric" in res:
# lyric = res["lyric"]
# # decode
# lyric = html.unescape(lyric)
# lyric = html.unescape(lyric)
# lyric = parse.unquote(lyric)
#
# it = re.finditer(r"\[(.*?)\](.+)", lyric)
# lyric_lines = []
# for match in it:
# time_pop_up = match.group(1)
# time_split = time_pop_up.split(".")
# ms = float("0.{}".format(time_split[1]))
# sec = time.strptime(time_split[0], "%M:%S").tm_sec
# line = match.group(2)
# line = line.strip()
# if re.search("[:：]", line) or line == "" or line == "此歌曲为没有填词的纯音乐，请您欣赏":
# continue
# lyric_lines.append({
# "time": sec + ms,
# "line": line,
# })
#
# return lyric_lines[1:]
# else:
# return []
if "lyric" in res:
lyric = res["lyric"]
if "此歌曲为没有填词的纯音乐，请您欣赏" in lyric:
return {}, []
# decode
lyric = html.unescape(lyric)
lyric = html.unescape(lyric)
lyric = parse.unquote(lyric)
it = re.finditer(r"\[(\d+):(\d+.\d+)\](.+)", lyric)
lyric_lines = []
contributors_dict = {}
for match in it:
min = float(match.group(1))
try:
sec = float(match.group(2))
except ValueError:
sec = 0
line = match.group(3)
line = line.strip()
if line == "":
continue
se_contributors = re.search("(.*?)[:：](.*)", line)
if se_contributors:
contributors_dict[se_contributors.group(1).strip()] = se_contributors.group(2).strip()
continue
lyric_lines.append({
"time": min * 60 + sec,
"line": line,
})
return contributors_dict, lyric_lines[1:]
else:
return {}, []
def get_entity(query_str, get_proxies_fun, wait=1.5):
rel_org_name_set = set()
logging.warning("start crawling {}...".format(query_str))
text = google_search(query_str, get_proxies_fun)
random.seed(time.time())
time.sleep(wait * random.random())
soup = BeautifulSoup(text, "lxml")
# is there an entity in google KG?
div_kg_hearer = soup.select_one("div.kp-header")
if div_kg_hearer is None: # if there is no knowledge graph at the right, drop it
logging.warning("no entity returned for this query")
return None
enti_name = div_kg_hearer.select_one("div[role=heading] span")
enti_name = enti_name.text if enti_name is not None else None
if enti_name is None or "..." in enti_name:
se = re.search(
'\["t-dhmk9MkDbvI",.*\[\["data",null,null,null,null,\[null,"\[\\\\"(.*)\\\\",',
text)
if se is not None:
enti_name = se.group(1)
else:
logging.warning(
"sth went wrong when extracting the name of the entity")
return None
# identify the type
span_list = div_kg_hearer.select("span")
enti_type = span_list[-1].text if len(span_list) > 1 else "unknown"
# description from wikipedia
des = soup.find("h3", text="Description")
des_info = ""
if des is not None:
des_span = des.parent.select_one("span")
des_info = des_span.text if des_span is not None else ""
# extract attributes
attr_tags = soup.select("div.Z1hOCe")
attr_dict = {}
for attr in attr_tags:
attr_str = attr.get_text()
se = re.search("(.*?)[:：](.*)", attr_str)
if se is None:
continue
key_attr = se.group(1)
val_attr = se.group(2)
attr_dict[key_attr] = val_attr
# relevant org name on current page
a_reltype_list = soup.select("div.MRfBrb > a")
for a in a_reltype_list:
rel_org_name_set.add(a["title"].strip())
# collect next urls e.g. : more x+
div_list = soup.select("div.yp1CPe")
next = []
host = "https://www.google.com"
for div in div_list:
a_list = div.select("a.EbH0bb")
for a in a_list:
if "http" not in a["href"]:
next.append("%s%s" % (host, a["href"]))
# crawl parent org
a_parent_org = soup.find("a", text="Parent organization")
if a_parent_org is not None:
parent_str = a_parent_org.parent.parent.text.strip()
parent_org = parent_str.split(":")[1]
rel_org_name_set.add(parent_org.strip())
# crawl subsidiaries
a_subsidiaries = soup.find("a", text="Subsidiaries")
if a_subsidiaries is not None:
href = a_subsidiaries["href"]
if "http" not in href:
subsidiaries_str = a_subsidiaries.parent.parent.text.strip()
subs = subsidiaries_str.split(":")[1].split(",")
for sub in subs:
sub = sub.strip()
if sub == "MORE":
continue
rel_org_name_set.add(sub)
next.append("%s%s" % (host, href))
# scrawl urls in list 'next'
for url in tqdm(next, desc="crawling relevant org names..."):
res = requests_dora.try_best_2_get(
url,
invoked_by="get_org_name",
headers=requests_dora.get_default_headers(),
get_proxies_fun=get_proxies_fun)
soup = BeautifulSoup(res.text, "lxml")
# crawl items at the top
a_list = soup.select("a.klitem")
for a in a_list:
rel_org_name = a["title"]
rel_org_name_set.add(rel_org_name.strip())
# crawl headings under the map if any
heading_list = soup.select("div.VkpGBb")
for heading in heading_list:
heading_str = heading.select_one("div[role='heading']")
rel_org_name_set.add(heading_str.get_text())
random.seed(time.time())
bar.update()
time.sleep(wait * random.random())
rel_org_name_list = [
org_name for org_name in rel_org_name_set if len(org_name) > 1
]
return {
"query_str": query_str,
"name": enti_name,
"type": enti_type,
"des": des_info,
"attributes": attr_dict,
"rel_org": rel_org_name_list
}