async def get_8xs_some_label_all_video_list_by_label_name(
self, label_name='大陆') -> list:
"""
根据label_name 获取指定的所有视频信息
:param label_name:
:return:
"""
def get_tasks_params_list():
tasks_params_list = []
sort_name_dict = self.get_8xs_all_sort_label_name_dict(
sort_name=label_name)
max_page_num = sort_name_dict.get('max_page_num', 0)
sort_type = sort_name_dict.get('sort_type', '')
assert sort_type != ''
for page_num in range(1, max_page_num + 1):
tasks_params_list.append({
'page_num': page_num,
'label_name': label_name,
'sort_type': sort_type,
})
return tasks_params_list
def get_create_task_msg(k) -> str:
return 'create task[where type: {}, page_num: {}] ...'.format(
k['label_name'],
k['page_num'],
)
def get_now_args(k) -> list:
return [
k['label_name'],
k['sort_type'],
k['page_num'],
]
all_res = await get_or_handle_target_data_by_task_params_list(
loop=self.loop,
tasks_params_list=get_tasks_params_list(),
func_name_where_get_create_task_msg=get_create_task_msg,
func_name=self.get_8xs_video_list_by_label_name_and_page_num,
func_name_where_get_now_args=get_now_args,
func_name_where_handle_one_res=None,
func_name_where_add_one_res_2_all_res=default_add_one_res_2_all_res,
one_default_res=[],
step=self.concurrency,
logger=self.lg,
get_all_res=True,
concurrent_type=self.concurrent_type,
)
pprint(all_res)
print('获取{} video_all_res_num: {}'.format(label_name, len(all_res)))
all_res = list_remove_repeat_dict_plus(
target=all_res,
repeat_key='video_name',
)
print('实际获取{} video_all_res_num: {}'.format(label_name, len(all_res)))
return all_res
def _get_yw_one_type_company_id_list_task(self, ip_pool_type, keyword, page_num, timeout=15):
"""
获取yw某关键字的单页company_info(m 站)
:param self:
:param ip_pool_type:
:param keyword:
:param page_num:
:param timeout:
:return:
"""
headers = get_random_headers(
user_agent_type=1,
cache_control='',)
headers.update({
# 'x-csrf-token': 'v8N2st76hSgzPPYQ-1DYgqOh',
# 'Referer': 'http://wap.yiwugo.com/search?q=%E5%8E%8B%E7%BC%A9%E6%9C%BA',
'X-Requested-With': 'XMLHttpRequest',
})
params = (
('q', str(keyword)),
('cpage', str(page_num)),
('pageSize', '28'),
('st', '0'),
('m', ''),
('f', ''),
('s', ''),
)
s_url = 'http://wap.yiwugo.com/api/search/s.htm'
body = Requests.get_url_body(
url=s_url,
headers=headers,
params=params,
ip_pool_type=ip_pool_type,
num_retries=6,
timeout=timeout,)
# lg.info(body)
data = json_2_dict(
json_str=body,
default_res={},
logger=lg).get('prslist', [])
# lg.info(str(data))
company_info_list = [{
'company_id': item.get('shopUrlId', ''),
'company_name': item.get('shopName', ''),
} for item in data]
company_info_list = list_remove_repeat_dict_plus(target=company_info_list, repeat_key='company_id')
# lg.info(str(company_info_list))
lg.info('[{}] keyword: {}, page_num: {}'.format(
'+' if company_info_list != [] else '-',
keyword,
page_num,))
collect()
return company_info_list
def get_bd_shop_info():
ak = get_ak()
# ak = ''
# 百度api 关键字搜索信息
tmp_shop_list = get_bd_map_shop_info_list_by_keyword_and_area_name(
ak=ak,
keyword='官方',
area_name='杭州',
page_num=2,
ip_pool_type=tri_ip_pool,
timeout=15,
num_retries=8,)
# pprint(tmp_shop_list)
shop_info_list = []
for item in tmp_shop_list:
try:
phone = item.get('telephone', '')
assert phone != '', 'phone不为空str!'
phone = [{
'phone': item.replace('(', '').replace(')', ''),
} for item in phone.split(',')]
address = item.get('address', '')
assert address != '', 'address不为空str'
company_name = item.get('name', '')
assert company_name != '', 'company_name不为空str!'
city_name = item.get('city', '')
assert city_name != '', 'city_name != ""'
province_name = item.get('province', '')
assert province_name != '', 'province_name != ""'
company_id = item.get('uid', '')
assert company_id != '', 'company_id != ""'
lat = item.get('location', {}).get('lat', 0.)
lng = item.get('location', {}).get('lng', 0.)
assert lat != 0. or lng != 0., 'lat or lng异常!'
except AssertionError:
continue
shop_info_list.append({
'company_id': company_id,
'company_name': company_name,
'address': address,
'city_name': city_name,
'province_name': province_name,
'phone': phone,
'lat': lat,
'lng': lng,
})
shop_info_list = list_remove_repeat_dict_plus(
target=shop_info_list,
repeat_key='company_id',)
pprint(shop_info_list)
def _parse_someone_cate_id_api_info(
self,
target_list: list,
_type='m',
) -> list:
"""
解析m站, pc站接口的信息
:param target_list:
:return:
"""
# pprint(target_list)
res = []
for item in target_list:
try:
title = item.get('title', '')
assert title != '', 'title != ""'
share_num = item.get('share_num', '0')
assert share_num != '0', "share_num != '0'"
id = item.get('id', )
assert id is not None, 'id is not None'
if _type == 'm':
read_num = item.get('read_num', '0')
elif _type == 'pc':
read_num = item.get('read_sum', '0')
else:
raise NotImplemented
assert read_num != '0', "read_num != '0'"
except (AssertionError, Exception) as e:
# print('遇到错误:', e)
continue
res.append({
'id':
id,
'title':
title,
'read_num':
read_num,
'share_num':
share_num,
'url':
'https://focus.youth.cn/mobile/detail/id/{}#'.format(id)
})
res = list_remove_repeat_dict_plus(
target=res,
repeat_key='title',
)
# 按阅读数正序排列
# res = sorted(res, key=lambda item: int(item.get('read_num', '0')), reverse=True)
return res
async def get_s69_all_chinese_captions_video_list(self) -> list:
"""
获取所有中文字幕的video list
:return:
"""
def get_tasks_params_list():
tasks_params_list = []
for page_num in range(1,
self.max_s63_chinese_captions_page_num + 1):
tasks_params_list.append({
'page_num': page_num,
})
return tasks_params_list
def get_create_task_msg(k) -> str:
return 'create task[where type: {}, page_num: {}] ...'.format(
'中文字幕',
k['page_num'],
)
def get_now_args(k) -> list:
return [
k['page_num'],
]
all_res = await get_or_handle_target_data_by_task_params_list(
loop=self.loop,
tasks_params_list=get_tasks_params_list(),
func_name_where_get_create_task_msg=get_create_task_msg,
func_name=self.get_s69_chinese_captions_video_list_by_page_num,
func_name_where_get_now_args=get_now_args,
func_name_where_handle_one_res=None,
func_name_where_add_one_res_2_all_res=default_add_one_res_2_all_res,
one_default_res=[],
step=self.concurrency,
logger=self.lg,
get_all_res=True,
concurrent_type=self.concurrent_type,
)
pprint(all_res)
print('获取中文字幕video_all_res_num: {}'.format(len(all_res)))
# 不采用request, 不管page_num怎么变都只拿到首页数据, 改用driver
all_res = list_remove_repeat_dict_plus(
target=all_res,
repeat_key='video_name',
)
print('实际获取中文字幕video_all_res_num: {}'.format(len(all_res)))
return all_res
async def _search(self, k: str) -> list:
"""
搜索
:param k:
:return:
"""
# res = await self._search_by_finer(k=k)
res = await self._search_by_wkb_and_lyoo(k=k)
res = list_remove_repeat_dict_plus(
target=res,
repeat_key='question_desc',)
pprint(res)
self.lg.info('本次获取到k: {}, 结果数: {}'.format(
k,
len(res),
))
return res
def _get_bd_or_gd_one_type_company_info_list_task(self,
ak:str,
keyword,
area_name,
page_num:int,
ip_pool_type,
timeout=15,
num_retries=8,
map_type='bd') -> list:
"""
获取bd or gd的商家信息
:param self:
:param ak: 申请的ak or key码
:param keyword: eg: '鞋子'
:param area_name: eg: '金华市'
:param page_num:
:param ip_pool_type:
:param timeout:
:param num_retries:
:param map_type: 'bd' or 'gd'
:return:
"""
def get_bd() -> list:
"""bd"""
# 百度api 关键字搜索信息
tmp_shop_list = get_bd_map_shop_info_list_by_keyword_and_area_name(
ak=ak,
keyword=keyword,
area_name=area_name,
page_num=page_num,
ip_pool_type=ip_pool_type,
timeout=timeout,
num_retries=num_retries,
logger=lg,)
# pprint(tmp_shop_list)
shop_info_list = []
for item in tmp_shop_list:
try:
phone = item.get('telephone', '')
assert phone != '', 'phone不为空str!'
phone = [{
'phone': item.replace('(', '').replace(')', ''),
} for item in phone.split(',')]
address = item.get('address', '')
assert address != '', 'address不为空str'
company_name = item.get('name', '')
assert company_name != '', 'company_name不为空str!'
city_name = item.get('city', '')
assert city_name != '', 'city_name != ""'
province_name = item.get('province', '')
assert province_name != '', 'province_name != ""'
company_id = item.get('uid', '')
assert company_id != '', 'company_id != ""'
lat = item.get('location', {}).get('lat', 0.)
lng = item.get('location', {}).get('lng', 0.)
assert lat != 0. or lng != 0., 'lat or lng异常!'
except AssertionError:
continue
shop_info_list.append({
'company_id': company_id,
'company_name': company_name,
'address': address,
'city_name': city_name,
'province_name': province_name,
'phone': phone,
'lat': lat,
'lng': lng,
})
return shop_info_list
def get_gd() -> list:
"""gd"""
tmp_shop_info_data = get_gd_map_shop_info_list_by_keyword_and_area_name(
gd_key=ak,
keyword=str(keyword),
area_name=str(area_name),
page_num=page_num,
ip_pool_type=ip_pool_type,
num_retries=num_retries,
timeout=timeout,
logger=lg,)
shop_info_list = []
for item in tmp_shop_info_data:
try:
company_id = item.get('id', '')
assert company_id != '', 'company_id != ""'
company_name = item.get('name', '')
assert company_name != '', 'company_name != ""'
address = item.get('address', '')
assert address != '', "address != ''"
city_name = item.get('cityname', '')
assert city_name != '', 'city_name != ""'
province_name = item.get('pname', '')
assert province_name != '', 'province_name != ""'
phone = item.get('tel', '')
assert phone != '', 'phone != ""'
phone = [{
'phone': item,
} for item in phone.split(';')]
location = item.get('location', '')
assert location != '', 'localtion != ""'
# 经度
lng = float(location.split(',')[0])
# 纬度
lat = float(location.split(',')[1])
except (AssertionError, Exception):
continue
shop_info_list.append({
'company_id': company_id,
'company_name': company_name,
'address': address,
'city_name': city_name,
'province_name': province_name,
'phone': phone,
'lat': lat,
'lng': lng,
})
return shop_info_list
if map_type == 'bd':
shop_info_list = get_bd()
elif map_type == 'gd':
shop_info_list = get_gd()
else:
raise NotImplementedError
shop_info_list = list_remove_repeat_dict_plus(
target=shop_info_list,
repeat_key='company_id',)
# pprint(shop_info_list)
lg.info('[{}] keyword:{}, page_num:{}, area_name: {}'.format(
'+' if shop_info_list != [] else '-',
keyword,
page_num,
area_name,))
return shop_info_list
def _get_gt_one_type_company_id_list_task(self,
ip_pool_type,
keyword,
company_url_selector:dict,
company_id_selector:dict,
page_num,
num_retries=8,
timeout=15,) -> list:
"""
根据keyword获取gt单页的所有comapny_id list
:param self:
:param ip_pool_type:
:param keyword:
:param page_num:
:param num_retries:
:param timeout:
:return:
"""
# search
headers = get_random_headers()
headers.update({
# 'Referer': 'http://www.go2.cn/search/all/?category_id=all&search_1=1&q=%E9%9E%8B%E5%AD%90',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
})
params = (
('category_id', 'all'),
('search_1', '1'),
('q', str(keyword)),
)
url = 'http://www.go2.cn/search/all/page{}.html'.format(page_num)
body = Requests.get_url_body(
url=url,
headers=headers,
params=params,
ip_pool_type=ip_pool_type,
num_retries=num_retries,
timeout=timeout,)
# lg.info(body)
company_url_list = parse_field(
parser=company_url_selector,
target_obj=body,
is_first=False,
logger=lg,)
# pprint(company_url_list)
res = []
for item in company_url_list:
try:
company_id = parse_field(
parser=company_id_selector,
target_obj=item,
logger=lg,)
assert company_id != '', 'company_id不为空值!'
except AssertionError:
continue
res.append({
'company_id': company_id,
})
res = list_remove_repeat_dict_plus(
target=res,
repeat_key='company_id',)
lg.info('[{}] keyword: {}, page_num: {}'.format(
'+' if res != [] else '-',
keyword,
page_num,))
return res
def _get_pk_one_type_company_id_list_task(self,
ip_pool_type,
keyword:str,
page_num,
province_name,
city_name,
city_id,
w3,
num_retries=6,
timeout=15,) -> list:
"""
获取pk单个关键字单页的company_id_list
:param self:
:param ip_pool_type:
:param keyword:
:param page_num:
:param province_name:
:param city_name:
:param city_id:
:param w3:
:param num_retries:
:param timeout:
:return:
"""
headers = get_random_headers(
user_agent_type=1,
connection_status_keep_alive=False,
cache_control='',)
headers.update({
'accept': 'application/json, text/plain, */*',
'Origin': 'https://m.ppkoo.com',
})
params = (
# ('cid', '50000436'), # 根据keywords索引的话, cid可不传
('keywords', keyword),
('hot', 'desc'),
('page', str(page_num)),
('city_id', str(city_id)),
# ('v', '3784143914913054'),
)
url = 'https://www.ppkoo.com/api/Search/goods'
body = Requests.get_url_body(
url=url,
headers=headers,
params=params,
ip_pool_type=ip_pool_type,
num_retries=num_retries,
timeout=timeout,)
# 存在: {"status":true,"total":"0","data":null}
# lg.info(body)
data = json_2_dict(
json_str=body,
default_res={},
logger=lg).get('data', [])
# pprint(data)
if data is None:
# 处理null的赋值情况
data = []
else:
pass
company_info_list = []
for item in data:
try:
company_id = item.get('business_id', '')
assert company_id != ''
address = item.get('shop_location', '')
assert address != ''
company_info_list.append({
'company_id': company_id,
'province_name': province_name,
'city_name': city_name,
'w3': w3,
'address': address, # 此处即可获取详细地址!
})
except Exception:
continue
company_info_list = list_remove_repeat_dict_plus(target=company_info_list, repeat_key='company_id')
# pprint(company_info_list)
lg.info('[{}] keyword: {}, page_num: {}'.format(
'+' if company_info_list != [] else '-',
keyword,
page_num,))
collect()
return company_info_list
def _get_hn_one_type_company_id_list_task(self,
ip_pool_type,
keyword, page_num,
province_name,
city_name,
city_base_url,
shop_item_selector:dict,
shop_id_selector:dict,
w3_selector:dict,
num_retries=6,
timeout=15,):
"""
获取hn 某关键字的单页company_info
:param self:
:param ip_pool_type:
:param keyword:
:param page_num:
:param province_name:
:param city_name:
:param city_base_url:
:param shop_item_selector:
:param shop_id_selector:
:param w3_selector:
:param num_retries:
:param timeout:
:return: [{'company_id': xxx, 'province_name': 'xx', 'city_name': 'xx', 'w3': 'xx'}, ...]
"""
try:
w3 = parse_field(
parser=w3_selector,
target_obj=city_base_url,
logger=lg,)
assert w3 != '', 'w3为空值!'
except AssertionError:
lg.error('遇到错误:', exc_info=True)
return []
headers = get_random_headers()
headers.update({
'Proxy-Connection': 'keep-alive',
})
params = (
('q', str(keyword)),
('sourcePage', '/'),
('page_no', str(page_num)),
)
# url = 'http://www.huoniuniu.com/goods'
url = city_base_url + '/goods'
body = Requests.get_url_body(
url=url,
headers=headers,
params=params,
num_retries=num_retries,
ip_pool_type=ip_pool_type,
timeout=timeout)
# lg.info(body)
shop_item_list = parse_field(
parser=shop_item_selector,
target_obj=body,
is_first=False,
logger=lg,)
# pprint(shop_item_list)
shop_id_list = []
for item in shop_item_list:
try:
company_id = parse_field(
parser=shop_id_selector,
target_obj=item,
is_first=True,
logger=lg)
assert company_id != '', 'company_id不为空值!'
shop_id_list.append({
'company_id': company_id,
'province_name': province_name,
'city_name': city_name,
'w3': w3,
})
except AssertionError:
continue
shop_id_list = list_remove_repeat_dict_plus(
target=shop_id_list,
repeat_key='company_id',)
# pprint(shop_id_list)
lg.info('[{}] keyword: {}, page_num: {}, province_name: {}, city_name: {}'.format(
'+' if shop_id_list != [] else '-',
keyword,
page_num,
province_name,
city_name,
))
collect()
return shop_id_list