def get_releaser_follower_num(self, releaserUrl):
if "www.yixia.com" in releaserUrl:
get_page = retry_get_url(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
midstep_1 = soup.find('ul', {'class': 'bottomInfor'})
midstep_2 = midstep_1.find_all('li')
for line in midstep_2:
line_text = line.text
if '粉丝' in line_text:
follower_str = line_text.replace('粉丝', '')
follower_num = trans_play_count(follower_str)
print('%s follower number is %s' % (releaserUrl, follower_num))
return follower_num
except:
print("can't can followers")
elif "n.miaopai.com" in releaserUrl:
try:
split_url = releaserUrl.split("personal/")
suid = split_url[-1].replace('.htm', '').replace('.html',
'').replace(
'htm', '')
url = "https://n.miaopai.com/api/aj_user/space.json?suid=%s" % suid
get_page = urllib.request.urlopen(url)
page_bytes = get_page.read()
page_str = page_bytes.decode("utf-8")
page_dic = json.loads(page_str)
follower_num = page_dic['data']['followers_count']
return follower_num
except:
print("can't can followers")
def get_releaser_info(self, releaser_id):
url = 'https://mguser.api.max.mgtv.com/artist/getArtistInfo?uid=' + releaser_id
get_page = retry_get_url(url)
get_page.encoding = 'utf-8'
page = get_page.text
page_dic = json.loads(page)
releaser = page_dic['data']['nickName']
return releaser
def search_page(self,
keyword=None,
search_pages_max=30,
output_to_es_raw=False,
output_to_es_register=False,
es_index=None,
doc_type=None):
list_page = []
count = 0
pages = 0
while pages < search_pages_max:
pages += 1
try:
url = ('http://app.pearvideo.com/clt/jsp/v3/search.jsp?k=' +
keyword + '&start=' + str(count))
get_page = retry_get_url(url, headers=self.headers)
get_page.encoding = 'utf-8'
page = get_page.text
print('get page done')
page = page.replace('\n', '')
page = page.replace('\r', '')
page_dic = json.loads(page)
midstep = page_dic['searchList']
count += 10
for line in midstep:
try:
contid = line['contId']
line_dic = self.video_page_by_video_id(contid)
print('get one line done')
list_page.append(line_dic)
if len(list_page) >= 100:
output_result(
result_Lst=list_page,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
list_page.clear()
except:
pass
except:
pass
if list_page != []:
output_result(result_Lst=list_page,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
return list_page
def video_page(self, url):
find_video_id = ' '.join(re.findall('video_\d+', url))
video_id = ' '.join(re.findall('\d+', find_video_id))
real_url = 'http://app.pearvideo.com/clt/jsp/v3/content.jsp?contId=' + video_id
get_page = retry_get_url(real_url, headers=self.headers)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('\n', '')
page = page.replace('\r', '')
page_dic = json.loads(page)
if page_dic['resultMsg'] == 'success':
self.video_data['title'] = page_dic['content']['name']
self.video_data['video_id'] = video_id
self.video_data['url'] = url
self.video_data['releaser'] = page_dic['content']['authors'][0][
'nickname']
releaser_id = page_dic['content']['authors'][0]['userId']
self.video_data['releaser_id'] = releaser_id
self.video_data[
'releaserUrl'] = 'http://www.pearvideo.com/author_' + releaser_id
self.video_data['comment_count'] = page_dic['content'][
'commentTimes']
self.video_data['favorite_count'] = page_dic['content'][
'praiseTimes']
rt_time = page_dic['content']['pubTime']
self.video_data['release_time'] = int(
datetime.datetime.strptime(rt_time,
'%Y-%m-%d %H:%M').timestamp() * 1e3)
dura = page_dic['content']['duration']
dura = dura.replace('"', '')
dura_lst = dura.split("'")
self.video_data['duration'] = (int(dura_lst[0])) * 60 + int(
dura_lst[1])
self.video_data['fetch_time'] = int(
datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
video_info = copy.deepcopy(self.video_data)
else:
print('error' + url)
return video_info
def get_info_from_video_page(self, url):
video_lst = []
id_lst = []
get_page = retry_get_url(url)
get_page.encoding = 'utf-8'
page = get_page.text
try:
releaser_id = re.findall('puuid:.*', page)[0].split('"')[1]
except:
releaser_id = None
if len(releaser_id) > 5:
releaserUrl = 'https://www.mgtv.com/u/' + releaser_id + '/d.html'
releaser = self.get_releaser_info(releaser_id)
else:
releaserUrl = None
releaser = None
vid = re.findall('\d+', re.findall('vid:.*', page)[0])[0]
video_info_url = ('https://pcweb.api.mgtv.com/common/list?video_id=' +
vid + '&cxid=&version=5.5.35')
get_video_page = retry_get_url(video_info_url)
get_video_page.encoding = 'utf-8'
video_page = get_video_page.text
video_page_dic = json.loads(video_page)
try:
normal_video_lst = video_page_dic['data']['list']
except:
normal_video_lst = None
try:
short_video_lst = video_page_dic['data']['short']
except:
short_video_lst = None
if normal_video_lst is not None:
for line in normal_video_lst:
title = line['t1']
url = 'https://www.mgtv.com' + line['url']
play_count = line['playcnt']
video_id = line['video_id']
clip_id = line['clip_id']
rt_or_dura_str = line['t2']
try:
release_time = int(
datetime.datetime.strptime(
rt_or_dura_str, '%Y-%m-%d').timestamp() * 1e3)
duration = 0
except:
release_time = 0
duration = self.trans_dura_str_to_dura_int(rt_or_dura_str)
else:
release_time = 0
duration = 0
fetch_time = int(
datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
self.video_data['title'] = title
self.video_data['url'] = url
self.video_data['play_count'] = play_count
self.video_data['releaser'] = releaser
self.video_data['releaserUrl'] = releaserUrl
if release_time != 0:
self.video_data['release_time'] = release_time
if duration != 0:
self.video_data['duration'] = duration
self.video_data['fetch_time'] = fetch_time
self.video_data['video_id'] = video_id
self.video_data['clip_id'] = clip_id
id_lst.append(video_id)
get_data = copy.deepcopy(self.video_data)
video_lst.append(get_data)
print('get_one_long_video')
if short_video_lst is not None:
for line in short_video_lst:
title = line['t1']
url = 'https://www.mgtv.com' + line['url']
play_count = line['playcnt']
video_id = line['video_id']
clip_id = line['clip_id']
rt_or_dura_str = line['t2']
try:
release_time = int(
datetime.datetime.strptime(
rt_or_dura_str, '%Y-%m-%d').timestamp() * 1e3)
duration = 0
except:
release_time = 0
duration = self.trans_dura_str_to_dura_int(rt_or_dura_str)
else:
release_time = 0
duration = 0
fetch_time = int(
datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
self.video_data['title'] = title
self.video_data['url'] = url
self.video_data['play_count'] = play_count
self.video_data['releaser'] = releaser
self.video_data['releaserUrl'] = releaserUrl
if release_time != 0:
self.video_data['release_time'] = release_time
if duration != 0:
self.video_data['duration'] = duration
self.video_data['fetch_time'] = fetch_time
self.video_data['video_id'] = video_id
self.video_data['clip_id'] = clip_id
id_lst.append(video_id)
get_data = copy.deepcopy(self.video_data)
video_lst.append(get_data)
print('get_one_short')
video_info_tup = (id_lst, video_lst)
return video_info_tup
def search_page(self,
keyword,
search_pages_max=30,
output_to_es_raw=False,
output_to_es_register=False,
es_index=None,
doc_type=None):
processing_lst = []
result_lst = []
id_lst = []
video_info_lst = []
count = 1
while count <= search_pages_max:
search_url = 'https://so.mgtv.com/so/k-' + keyword + '?page=' + str(
count)
count += 1
get_page = retry_get_url(search_url)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
video_lst = soup.find_all(
'div', {'class': 'so-result-info search-shortfilm clearfix'})
for line in video_lst:
title = line.find('img')['alt']
url = 'https:' + line.find(
'a', {'class': 'report-click report-action'})['href']
video_id = line.find(
'a', {'class': 'report-click report-action'})['video-id']
clip_id = line.find(
'a', {'class': 'report-click report-action'})['clip-id']
dura_str = line.find('span', {'class': 'rb'}).text
duration = self.trans_dura_str_to_dura_int(dura_str)
release_time_str = (line.find('span', {
'class': 'sho'
}).text.replace('发布时间：', '').replace('\n',
'').replace(' ', ''))
release_time = int(
datetime.datetime.strptime(release_time_str,
'%Y-%m-%d').timestamp() * 1e3)
fetch_time = int(
datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
self.video_data['title'] = title
self.video_data['url'] = url
self.video_data['duration'] = duration
self.video_data['release_time'] = release_time
self.video_data['fetch_time'] = fetch_time
self.video_data['video_id'] = video_id
self.video_data['clip_id'] = clip_id
get_data = copy.deepcopy(self.video_data)
processing_lst.append(get_data)
print('get_one_inmature_video')
print('end of search_page')
if video_info_lst == []:
url = processing_lst[0]['url']
print(url)
video_tup = self.get_info_from_video_page(url)
id_lst_son = video_tup[0]
video_lst_son = video_tup[1]
for line in id_lst_son:
id_lst.append(line)
for line in video_lst_son:
video_info_lst.append(line)
for line in processing_lst:
if line['video_id'] in id_lst:
for one_video in video_info_lst:
if line['video_id'] == one_video['video_id']:
line['play_count'] = one_video['play_count']
line['title'] = one_video['title']
line['releaser'] = one_video['releaser']
line['releaserUrl'] = one_video['releaserUrl']
result_lst.append(line)
print('get_one_video')
if len(result_lst) >= 100:
output_result(
result_Lst=result_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
result_lst.clear()
else:
url = line['url']
video_tup = self.get_info_from_video_page(url)
id_lst_son = video_tup[0]
video_lst_son = video_tup[1]
for video_id in id_lst_son:
id_lst.append(video_id)
for video_info in video_lst_son:
video_info_lst.append(video_info)
for one_video in video_info_lst:
if line['video_id'] == one_video['video_id']:
line['play_count'] = one_video['play_count']
line['title'] = one_video['title']
line['releaser'] = one_video['releaser']
line['releaserUrl'] = one_video['releaserUrl']
result_lst.append(line)
print('get_one_video')
if len(result_lst) >= 100:
output_result(
result_Lst=result_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
result_lst.clear()
if len(result_lst) != []:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
print('success')
return result_lst
def list_page(self,
list_name,
error_file=None,
output_to_file=False,
filepath=None,
output_to_es_raw=False,
output_to_es_register=False,
push_to_redis=False,
total_wanted_video_num=100):
category_id = self.category_corresponding[list_name]
result_Lst = []
count = 0
list_url = 'http://app.pearvideo.com/clt/jsp/v3/getCategoryConts.jsp?categoryId=' + category_id + '&start=0'
while list_url != '' and count < total_wanted_video_num:
get_page = retry_get_url(list_url, headers=self.headers)
get_page.encoding = 'utf-8'
page = get_page.text
print('get page done')
page = page.replace('\n', '')
page = page.replace('\r', '')
page_dic = json.loads(page)
list_url = page_dic['nextUrl']
video_lst_hot = page_dic['rankList']
video_lst_gen = page_dic['contList']
if video_lst_hot != []:
for line in video_lst_hot:
try:
contid = line['contId']
line_dic = self.video_page_by_video_id(contid)
print('get one line done')
result_Lst.append(line_dic)
except:
pass
for line in video_lst_gen:
try:
contid = line['contId']
line_dic = self.video_page_by_video_id(contid)
print('get one line done')
result_Lst.append(line_dic)
except:
pass
count += len(video_lst_gen) + len(video_lst_hot)
print(count)
if len(result_Lst) >= 100:
output_result(
result_Lst,
self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
)
result_Lst.clear()
if len(result_Lst) != []:
output_result(
result_Lst,
self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
)
result_Lst.clear()
def releaser_page(self,
releaserUrl,
error_file=None,
output_to_file=False,
filepath=None,
output_to_es_raw=False,
output_to_es_register=False,
push_to_redis=False,
releaser_page_num_max=30):
result_Lst = []
find_releaser_id = ' '.join(re.findall('author_\d+', releaserUrl))
releaser_id = ' '.join(re.findall('\d+', find_releaser_id))
real_releaserUrl = (
'http://app.pearvideo.com/clt/jsp/v3/userHome.jsp?userId=' +
releaser_id + '&reqType=1&start=0')
count = 0
while real_releaserUrl != '' and count < releaser_page_num_max * 10:
count += 10
get_page = retry_get_url(real_releaserUrl, headers=self.headers)
get_page.encoding = 'utf-8'
page = get_page.text
print('get page done')
page = page.replace('\n', '')
page = page.replace('\r', '')
page_dic = json.loads(page)
video_lst = page_dic['dataList']
real_releaserUrl = page_dic['nextUrl']
print('get next url:' + real_releaserUrl)
for line in video_lst:
try:
video_id = line['contInfo']['contId']
self.video_data['title'] = line['contInfo']['name']
self.video_data['video_id'] = video_id
self.video_data[
'url'] = 'http://www.pearvideo.com/video_' + video_id
self.video_data['releaser_id'] = releaser_id
self.video_data['releaserUrl'] = releaserUrl
self.video_data['favorite_count'] = line['contInfo'][
'praiseTimes']
rt_time = line['pubTime']
self.video_data['release_time'] = int(
datetime.datetime.strptime(
rt_time, '%Y-%m-%d %H:%M').timestamp() * 1e3)
dura = line['contInfo']['duration']
dura = dura.replace('"', '')
dura_lst = dura.split("'")
self.video_data['duration'] = (int(
dura_lst[0])) * 60 + int(dura_lst[1])
self.video_data['fetch_time'] = int(
datetime.datetime.timestamp(datetime.datetime.now()) *
1e3)
video_info = copy.deepcopy(self.video_data)
print('get one video_info')
result_Lst.append(video_info)
if len(result_Lst) >= 100:
output_result(
result_Lst,
self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
)
result_Lst.clear()
except:
pass
if len(result_Lst) != []:
output_result(
result_Lst,
self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
)
result_Lst.clear()