class Pipeline():
'''
存储结果管道
（不负责压next task）
'''
def __init__(self, task):
self.task = task
self.taskUtil = TaskUtil()
self.mongoUtil = MongoUtil()
pass
def run(self):
'''
分发
:return: 无
'''
Log.i('Pipeline.run()')
if self.task['results'] is not None and len(self.task['results']) > 0:
#下次任务入队列
if self.task['next_tasks'] is not None:
for next_task in self.task['next_tasks']:
self.taskUtil.insert_one(next_task)
#本次解析结果入库
# 利用反射机制自动执行pipeline_<parser名>（）函数，如果找不到则执行默认的pipeline_default()函数
if hasattr(self, 'pipeline_' + self.task['parser']):
func = getattr(self, 'pipeline_' + self.task['parser'])
func(self.task['table'])
else:
self.pipeline_default(self.task['table'])
#将完整task存入mongo，并将本条task
self.task['state'] = 'done'
self.taskUtil.replace_one(self.task['_id'], self.task)
else:
#没有解析出结果，则表示中间出错了，等待下次再启动
pass
Log.i('this task is finished')
def pipeline_default(self, collection_name):
'''
存储demo
demo_info
{
'_id':'http://tieba.baidu.com',
'name':'百度贴吧'
}
:return:
'''
if self.task['parent'] is None:
self.task['parent'] = {}
if self.task['results'] is not None:
for result in self.task['results']:
insert_data = dict(self.task['parent'], **result)
self.mongoUtil.insert(collection_name=collection_name,
insert_data=insert_data)
pass
def __del__(self):
self.mongoUtil.close_conn()
def __init__(self, request):
self.mongoUtil = MongoUtil()
count = self.mongoUtil.count(collection_name='tasks')
if count == 0:
task = None
task_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '..', 'conf',
'task.json')
f = open(task_file, encoding='utf-8')
task = f.read()
f.close()
insert_data = json.loads(task)
self.mongoUtil.insert(collection_name='tasks',
insert_data=insert_data)
class InitUtil:
'''
初始化，清空所有数据，重新开始新一轮任务
'''
def __init__(self):
self.mongoUtil = MongoUtil()
pass
def init(self):
db = Setting.MONGO_DB
self.mongoUtil.clear_all(db)
def __del__(self):
self.mongoUtil.close_conn()
def __init__(self):
self.mongoUtil = MongoUtil()
count = self.mongoUtil.count(collection_name='tasks')
if count == 0:
first_task_parser = Setting.FIRST_TASK_PARSER
first_task_url = Setting.FIRST_TASK_URL
first_task_table = Setting.FIRST_TASK_TABLE
insert_data = {
"parser": first_task_parser,
"request": first_task_url,
"table": first_task_table,
"parent": {},
"state": "ready"
}
self.mongoUtil.insert(collection_name='tasks',
insert_data=insert_data)
def __init__(self):
self.Mg = MongoUtil()
class Spider:
def __init__(self):
self.Mg = MongoUtil()
def run(self, site_id=None):
coll = self.Mg.getCol('site')
res = coll.find()
for site in res:
# try:
if 'data_type' in site and site['data_type'] == 'json':
self.update_site_json(site)
last_page = self.get_films_json(site)
else:
pass
self.update_site(site, coll)
last_page = self.get_films(site)
coll.update_one({'_id': site['_id']},
{'$set': {
'last_page': last_page
}})
# except:
# continue
# last_page = self.get_films(site)
# coll.update_one({'_id': site['_id']}, {'$set': {'last_page': last_page}})
# self.fixClassify()
# 过滤非法字符
def filter_xml(self, str):
str = str.strip()
for x in re_filters_xml:
str = re.sub(x, '', str)
return str
# 过滤电影
def filter_film(self, name):
if len(re.findall(re_filters_name, name)) > 0:
return 1
return 0
# 更新总数和pagesize
def update_site(self, site, coll):
try:
news = requests.get(site['api_url'], timeout=5)
root = ET.fromstring(self.filter_xml(news.text))
except:
return
_ns = root.find('list').attrib
tyClass = root.find('class')
configs = []
for x in tyClass:
configs.append({'ty': x.attrib['id'], 'key': x.text})
data = {
'last_recordcount': int(_ns['recordcount']),
'pagesize': int(_ns['pagesize']),
'pagecount': int(_ns['pagecount']),
'configs': configs
}
coll.update_one({'id': site['id']}, {"$set": data})
def set_classify(self, site, tid):
for x, y in site['classify'].items():
if int(tid) in y:
return int(x)
return 0
# 获取列表
def get_films(self, _site):
coll = self.Mg.getCol('films')
siteColl = self.Mg.getCol('site')
site = siteColl.find_one({'id': _site['id']})
# 补一页
page = site['last_page']
last = int(site['pagecount']) - site['last_page'] if int(
site['pagecount']) - site['last_page'] >= 0 else 0
addNum = 0
updateNum = 0
sameNum = 0
tqIt = tqdm(range(last * site['pagesize']),
total=last,
desc=self.set_desc(site, page, addNum, updateNum, sameNum))
for x in tqIt:
siteColl.update_one({'_id': site['_id']},
{'$set': {
'last_page': page
}})
url = site['api_url'] + '?ac=videolist&pg=%d' % page
page += 1
try:
res = requests.get(url, timeout=20)
except:
return page
try:
root = ET.fromstring(res.text.strip())
except:
continue
if len(root.find('list')) == 0:
return page
for child in root.find('list'):
last -= 1
film_name = child.find('name').text.upper() if child.find(
'name').text else ''
note = child.find('note').text.upper() if child.find(
'note').text else ''
video = []
for v in child.find('dl'):
flag = v.attrib['flag']
if flag == '' or not v.text or len(list(
v.text.split('#'))) == 0:
continue
video.append({
'key': flag,
'plist': list(v.text.split('#'))
})
if len(video) == 0:
continue
year = self.set_year(child.find('year').text)
info = {
'film_name':
film_name,
'tid':
int(child.find('tid').text),
'pic':
child.find('pic').text.strip()
if child.find('pic').text else '',
'site_id':
site['id'],
'state':
int(self.filter_film(film_name)),
'des':
child.find('des').text.strip()
if child.find('des').text else '',
'type':
child.find('type').text,
'area':
child.find('area').text,
'year':
int(year),
'douban_initial_year':
int(year),
'classify':
self.set_classify(site, int(child.find('tid').text)),
'note':
note,
'video':
video
}
if int(child.find('tid').text) not in site['filter_tid']:
continue
old = coll.find_one({'film_name': film_name, 'note': note})
if not old:
coll.insert_one(info)
addNum += 1
else:
sameNum += 1
oldVideo = old['video'] if 'video' in old else []
coll.update_one(
{'_id': old['_id']},
{'$set': {
'video': self.set_video(video, oldVideo)
}})
# new_video_keys = list(map(lambda x: x['key'], video))
# old_video_keys = list(map(lambda x: x['key'], old['video']))
# difference = list(set(new_video_keys).difference(set(old_video_keys)))
# for newKey in difference:
# newPlist = list(filter(lambda x: x['key'] == newKey, video))[0]
# coll.update_one({'_id': old['_id']}, {'$addToSet': {'video': newPlist}})
updateNum += 1
tqIt.set_description(
desc=self.set_desc(site, page, addNum, updateNum, sameNum))
return page
# 更新总数和pagesize
def update_site_json(self, site):
coll = self.Mg.getCol('site')
news = requests.get(site['api_url'], timeout=5)
news = news.json()
tyClass = news['class']
configs = []
for x in tyClass:
configs.append({'ty': x['type_id'], 'key': x['type_name']})
if 'total' in news:
data = {
'last_recordcount': int(news['total']),
'pagesize': int(news['limit']),
'pagecount': int(news['pagecount']),
'configs': configs
}
else:
page = news['page']
data = {
'last_recordcount': int(page['recordcount']),
'pagesize': int(page['pagesize']),
'pagecount': int(page['pagecount']),
'configs': configs
}
coll.update_one({'id': site['id']}, {"$set": data})
def get_films_json(self, _site):
coll = self.Mg.getCol('films')
siteColl = self.Mg.getCol('site')
site = siteColl.find_one({'id': _site['id']})
# 补一页
page = site['last_page']
# 剩余页数
last = int(site['pagecount']) - site['last_page']
addNum = 0
updateNum = 0
sameNum = 0
tqIt = tqdm(range(last * site['pagesize']),
desc=self.set_desc(site, page, addNum, updateNum, sameNum))
for x in tqIt:
siteColl.update_one({'_id': site['_id']},
{'$set': {
'last_page': page
}})
# 参数:p页码,翻页;wd关键词;cid分类id; 可以实现全部数据/分类数据获取还有搜索数据
url = site['api_url'] + '?ac=detail&pg=' + str(page)
page += 1
try:
res = requests.get(url, timeout=20)
root = res.json()
except:
return page
if len(root['list']) == 0:
return page
for child in root['list']:
last -= 1
film_name = child['vod_name'].upper(
) if child['vod_name'] else ''
note = child['vod_remarks'].upper(
) if child['vod_remarks'] else ''
kkey = child['vod_play_from'].split(
'$$$')[1] if child['vod_play_from'].find(
'$$$') > -1 else child['vod_play_from']
video = [{
'key': kkey,
'plist': child['vod_play_url'].split('#')
}]
if len(video) == 0 or int(
child['type_id']) not in site['filter_tid']:
continue
year = self.set_year(child['vod_year'])
info = {
'film_name':
film_name,
'tid':
child['type_id'],
'pic':
child['vod_pic'].strip() if child['vod_pic'] else '',
'site_id':
site['id'],
'state':
int(self.filter_film(film_name)),
'des':
child['vod_content'].strip()
if child['vod_content'] else '',
'type':
child['type_name'],
'area':
child['vod_area'],
'year':
int(year),
'douban_initial_year':
int(year),
'classify':
self.set_classify(site, int(child['type_id'])),
'note':
note,
'video':
video
}
old = coll.find_one({'film_name': film_name, 'note': note})
if not old:
coll.insert_one(info)
addNum += 1
else:
sameNum += 1
oldVideo = old['video'] if 'video' in old else []
coll.update_one(
{'_id': old['_id']},
{'$set': {
'video': self.set_video(video, oldVideo)
}})
updateNum += 1
tqIt.set_description(
desc=self.set_desc(site, page, addNum, updateNum, sameNum))
return page
def set_year(self, old):
old = re.sub(re.compile('普通话|年'), '', old)
year = int(old) if old and 1970 < int(old) < 2021 else 1970
return year
def set_video(self, newV, oldV):
new_video_keys = list(map(lambda x: x['key'], newV))
for x in oldV:
if x['key'] not in new_video_keys:
newV.append(x)
return newV
def set_desc(self, site, page, addNum, updateNum, sameNum):
return "[%s: %d, 总页数: %d, page: %d, 新增: %d, 修改: %d, 相同: %d]" % (
site['site_name'], site['id'], site['pagecount'], page, addNum,
updateNum, sameNum)
def fixClassify(self):
baseClassifyDict = {
"1":
[1, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, 24, 25, 26, 27, 29],
"2": [2, 12, 13, 14, 15, 16, 17, 18, 48, 54, 34, 33],
"3": [3, 45, 38, 37],
"4": [4, 41, 47, 28, 39, 40],
"5": [49, 51, 42, 43, 55, 19],
"6": [60, 44, 43, 42, 30],
"49": [49, 51, 52, 53, 55, 19]
}
coll = self.Mg.getCol('films')
site_list = self.Mg.getCol('site').find()
for site in site_list:
if 'type_change' in site:
for t in site['type_change']:
coll.update_many(
{
'site_id': site['id'],
'tid': int(t['ty']),
'new_tid': {
"$exists": False
}
}, {'$set': {
'new_tid': int(t['change'])
}})
for (k, v) in baseClassifyDict.items():
query = {
"$or": [{
u"$and": [{
u"new_tid": {
u"$exists": True
}
}, {
u"new_tid": {
u"$in": v
}
}]
}, {
u"$and": [{
u"new_tid": {
u"$exists": False
}
}, {
u"tid": {
u"$in": v
}
}]
}],
"classify": {
u"$exists": False
}
}
coll.update_many(query, {'$set': {'classify': k}})
class TaskUtil:
'''
Task操作工具
tasks
{
"parser": "phase"
"request": "http://so.eduyun.cn/synResource",
"response": "<html>...</html>"
"parent": {},
"state": "done",
"uptime":
}
'''
def __init__(self, request):
self.mongoUtil = MongoUtil()
count = self.mongoUtil.count(collection_name='tasks')
if count == 0:
task = None
task_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '..', 'conf',
'task.json')
f = open(task_file, encoding='utf-8')
task = f.read()
f.close()
insert_data = json.loads(task)
self.mongoUtil.insert(collection_name='tasks',
insert_data=insert_data)
def get_ready(self):
'''
获取一条待执行的任务（准备状态），并置为doing状态
:return: dict 单条任务
'''
# 过滤条件，不存在state字段或state=ready
filter_dict = {
'$or': [{
'state': {
'$exists': False
}
}, {
'state': 'ready'
}]
}
# 更新条件，将state=doing
update_dict = {'$set': {'state': 'doing'}}
# 执行mongo操作
task = self.mongoUtil.find_one_and_update(collection_name='tasks',
filter_dict=filter_dict,
update_dict=update_dict)
return task
def set_state(self, id, state):
'''
设置任务状态(ready，doing，done)
:param id: str 主键id
:param state: str 更新状态值
:return: 无
'''
filter_dict = {'_id': id}
update_dict = {'$set': {'state': state}}
self.mongoUtil.update(collection_name='tasks',
filter_dict=filter_dict,
update_dict=update_dict)
def replace_one(self, id, task):
'''
更新整个任务
:param id: str
:param task: dict
:return: 无
'''
filter_dict = {'_id': id}
r = self.mongoUtil.find_one_and_replace(collection_name='tasks',
filter_dict=filter_dict,
replace_dict=task)
return r
def insert_one(self, task):
'''
插入一条task
:param parser: str 解析器
:param request: str 请求的url
:return: 无
'''
r = self.mongoUtil.find_one(collection_name='tasks', filter_dict=task)
if r is None:
task['state'] = 'ready'
r = self.mongoUtil.insert(collection_name='tasks',
insert_data=task)
def __del__(self):
self.mongoUtil.close_conn()
from utils.MongoUtil import MongoUtil
Mg = MongoUtil()
col = Mg.getCol("site")
# site_config = {}
# for x in col.find():
# site_config[x['id']] = x['configs']
#
# print(site_config)
classify = {
'1': {
'1': [1, 5, 6, 7, 8, 9, 10, 11],
'2': [2, 12, 13, 14, 15, 16, 17, 18, 54],
'3': [4, 41, 39, 40, 47],
'49': [19, 49, 51, 52, 53, 55]
},
'2': {
'1': [1, 5, 6, 7, 8, 9, 10, 11, 24],
'2': [2, 12, 13, 14, 15, 17, 18, 20],
'3': [4, 41, 39, 40, 47],
'49': [16, 19.21, 22]
},
'3': {
'1': [1, 5, 6, 7, 8, 9, 10, 11],
'2': [2, 12, 13, 14, 15, 17, 18, 19, 21],
'3': [4],
'49': [16]
},
'4': {
class TaskUtil:
'''
Task操作工具
tasks
{
"parser": "phase"
"request": "http://so.eduyun.cn/synResource",
"response": "<html>...</html>"
"parent": {},
"state": "done",
"uptime":
}
'''
def __init__(self):
self.mongoUtil = MongoUtil()
count = self.mongoUtil.count(collection_name='tasks')
if count == 0:
first_task_parser = Setting.FIRST_TASK_PARSER
first_task_url = Setting.FIRST_TASK_URL
first_task_table = Setting.FIRST_TASK_TABLE
insert_data = {
"parser": first_task_parser,
"request": first_task_url,
"table": first_task_table,
"parent": {},
"state": "ready"
}
self.mongoUtil.insert(collection_name='tasks',
insert_data=insert_data)
def get_ready(self):
'''
获取一条待执行的任务（准备状态），并置为doing状态
:return: dict 单条任务
'''
# 过滤条件，不存在state字段或state=ready
filter_dict = {
'$or': [{
'state': {
'$exists': False
}
}, {
'state': 'ready'
}]
}
# 更新条件，将state=doing
update_dict = {'$set': {'state': 'doing'}}
# 执行mongo操作
task = self.mongoUtil.find_one_and_update(collection_name='tasks',
filter_dict=filter_dict,
update_dict=update_dict)
return task
def set_state(self, id, state):
'''
设置任务状态(ready，doing，done)
:param id: str 主键id
:param state: str 更新状态值
:return: 无
'''
filter_dict = {'_id': id}
update_dict = {'$set': {'state': state}}
self.mongoUtil.update(collection_name='tasks',
filter_dict=filter_dict,
update_dict=update_dict)
def replace_one(self, id, task):
'''
更新整个任务
:param id: str
:param task: dict
:return: 无
'''
filter_dict = {'_id': id}
r = self.mongoUtil.find_one_and_replace(collection_name='tasks',
filter_dict=filter_dict,
replace_dict=task)
return r
def insert_one(self, task):
'''
插入一条task
:param parser: str 解析器
:param request: str 请求的url
:return: 无
'''
r = self.mongoUtil.find_one(collection_name='tasks', filter_dict=task)
if r is None:
task['state'] = 'ready'
r = self.mongoUtil.insert(collection_name='tasks',
insert_data=task)
def __del__(self):
self.mongoUtil.close_conn()
def __init__(self, task):
self.task = task
self.taskUtil = TaskUtil()
self.mongoUtil = MongoUtil()
pass
def __init__(self):
self.mongoUtil = MongoUtil()
pass
def get_site():
Mg = MongoUtil()
siteColl = Mg.getCol('site')
res = siteColl.find({'id': 8})
print(list(res)[0])
def insert_site():
ss = {
'id':
34,
'site_key':
'okzy',
'api_url':
'https://api.okzy.tv/api.php/provide/vod/at/json/?ac=detail',
'site_name':
'okzy',
'configs': [{
'type_id': 1,
'type_name': '电影'
}, {
'type_id': 2,
'type_name': '连续剧'
}, {
'type_id': 3,
'type_name': '综艺'
}, {
'type_id': 4,
'type_name': '动漫'
}, {
'type_id': 5,
'type_name': '资讯'
}, {
'type_id': 6,
'type_name': '动作片'
}, {
'type_id': 7,
'type_name': '喜剧片'
}, {
'type_id': 8,
'type_name': '爱情片'
}, {
'type_id': 9,
'type_name': '科幻片'
}, {
'type_id': 10,
'type_name': '恐怖片'
}, {
'type_id': 11,
'type_name': '剧情片'
}, {
'type_id': 12,
'type_name': '战争片'
}, {
'type_id': 13,
'type_name': '国产剧'
}, {
'type_id': 14,
'type_name': '香港剧'
}, {
'type_id': 15,
'type_name': '韩国剧'
}, {
'type_id': 16,
'type_name': '欧美剧'
}, {
'type_id': 17,
'type_name': '公告'
}, {
'type_id': 18,
'type_name': '头条'
}, {
'type_id': 20,
'type_name': '纪录片'
}, {
'type_id': 21,
'type_name': '微电影'
}, {
'type_id': 22,
'type_name': '台湾剧'
}, {
'type_id': 23,
'type_name': '日本剧'
}, {
'type_id': 24,
'type_name': '海外剧'
}, {
'type_id': 25,
'type_name': '内地综艺'
}, {
'type_id': 26,
'type_name': '港台综艺'
}, {
'type_id': 27,
'type_name': '日韩综艺'
}, {
'type_id': 28,
'type_name': '欧美综艺'
}, {
'type_id': 29,
'type_name': '国产动漫'
}, {
'type_id': 30,
'type_name': '日韩动漫'
}, {
'type_id': 31,
'type_name': '欧美动漫'
}, {
'type_id': 32,
'type_name': '港台动漫'
}, {
'type_id': 33,
'type_name': '海外动漫'
}, {
'type_id': 34,
'type_name': '福利片'
}, {
'type_id': 35,
'type_name': '解说'
}, {
'type_id': 36,
'type_name': '电影解说'
}, {
'type_id': 37,
'type_name': '伦理片'
}],
'data_type':
'json',
'filter_tid': [3, 5, 17, 18, 20, 21, 22, 25, 26, 27, 28, 35, 36]
}
Mg = MongoUtil()
siteColl = Mg.getCol('site')
siteColl.insert_one(ss)
def __init__(self):
self.proxyArr = []
self.searchFilmList = []
self.film_coll = MongoUtil().getCol('films')
pass
class DoubanSpider(object):
def __init__(self):
self.proxyArr = []
self.searchFilmList = []
self.film_coll = MongoUtil().getCol('films')
pass
def run_search_link(self):
self.init_proxy(5)
# Db = MysqlHelper()
# self.searchFilmList = Db.select(
# "select id,film_name,douban_url from film_list where douban_url is null and classify in (1,2,4)")
query = {
"douban_rating": {
"$exists": False
},
"douban_state": {"$ne": 1},
"classify": {
"$in": [
1.0,
# 2.0,
# 4.0
]
}}
projection = {"_id": 1.0, 'film_name': 1}
self.searchFilmList = self.film_coll.find(query, projection=projection)
self.searchFilmList = list(self.searchFilmList)
self.count = len(self.searchFilmList)
print(self.count)
# asyncio.run(self.searchLink())
# 任务组， 最大协程数
loop = asyncio.get_event_loop()
tasks = []
for x in range(10):
tasks.append(self.searchLink())
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
# 搜索详情
def run_search_info(self):
self.init_proxy(2)
query = {'$and': [
{
"douban_url": {
u"$exists": True
}
},
{
"douban_rating": {
u"$exists": True
}
},
{
"douban_url": {
u"$ne": u"no"
}
},
{
"douban_country": {
u"$exists": False
}
},
{
"douban_state": {
u"$ne": 1
}
}
]}
self.searchFilmList = self.film_coll.find(query, projection={"_id": 1, 'douban_url': 1, 'year': 1})
self.searchFilmList = list(self.searchFilmList)
self.count = len(self.searchFilmList)
# asyncio.run(self.get_film_info())
pool = AsyncPool(maxsize=1)
for x in range(self.count):
pool.submit(self.get_film_info())
pool.release()
pool.wait()
# 初始化代理池
def init_proxy(self, num=2):
ip = requests.get('https://www.anpaitm.com/index/test?key=JGbeg4gFJD875obads')
requests.get(
'http://wapi.http.linkudp.com/index/index/save_white?neek=352139&appkey=3fdebe3814651abea473ec39b5dc53d1&white=' + ip.text)
res = requests.get(
f'http://webapi.http.zhimacangku.com/getip?num={num}&type=2&pro=&city=0&yys=0&port=11&time=1&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions=')
js = res.json()
for x in js['data']:
self.proxyArr.append(f"{x['ip']}:{x['port']}")
time.sleep(1)
# 获取代理, 随机拿一个
def get_proxy(self):
if len(self.proxyArr) == 0:
return ''
index = random.randint(0, len(self.proxyArr) - 1)
return self.proxyArr[index] if len(self.proxyArr) > 0 else ''
# 代理失效后的处理
def del_proxy(self, proxy):
if not proxy: return
if proxy in self.proxyArr:
self.proxyArr.remove(proxy)
if len(self.proxyArr) < 2:
self.init_proxy()
async def get_browser(self):
proxy = self.get_proxy()
browser = await launch({'headless': False, 'autoClose': True, 'args': [
f"--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{random.randint(56, 70)}.0.3538.25 Safari/537.36 Core/1.70.3861.400 QQBrowser/10.7.43{random.randint(0, 10)}.400",
'--proxy-server=%s' % proxy
]})
return browser, proxy
@staticmethod
def filter_film_name(name):
# 过滤电影名称
filter_re = re.compile("超清720P|高清\w*|%|\.|'|\\n| |\u3000\u3000|（\w+）|普通话版|粤语版|\((.*)\)")
return re.sub(filter_re, '', name).strip()
# def codingJson(self, data):
# return escape_string(self.filter_film_name(json.dumps(data, ensure_ascii=False)))
async def intercept_request(req):
"""请求过滤"""
if req.resourceType in ['image', 'media', 'eventsource', 'websocket']:
await req.abort()
else:
await req.continue_()
# 通过电影名称搜索豆瓣链接
async def searchLink(self):
browser = None
page = None
browser, proxy = await self.get_browser()
page = await browser.newPage()
# await page.setRequestInterception(True)
# page.on('request', "item=>self.intercept_request")
# page.on('request', lambda req: asyncio.ensure_future(self.intercept_request(req)))
while True:
await asyncio.sleep(random.uniform(0.5, 1.5))
if len(self.searchFilmList) == 0:
break
film = self.searchFilmList.pop()
film_name = self.filter_film_name(film['film_name'])
print(film_name)
link = 'https://search.douban.com/movie/subject_search?search_text=' + film_name
try:
goto_link = await page.goto(link, {'timeout': 50 * 1000})
body = await page.J('body')
body_text = await page.evaluate('item=>item.textContent', body)
if goto_link.status != 200 or body_text.find('检测到有异常') != -1 or body_text.find(
'服务异常') != -1 or body_text.find('请登录后再试') != -1:
self.del_proxy(proxy)
print('检测到有异常')
await page.close()
await browser.close()
await asyncio.sleep(random.uniform(0.5, 1.5))
browser, proxy = await self.get_browser()
page = await browser.newPage()
continue
# 没有找到
if body_text.find('换个搜索词试试吧') != -1 or body_text.find('根据相关法律法规和政策') != -1:
self.film_coll.update_one({'_id': film['_id']}, {'$set': {'douban_state': 1}})
print("\r剩余--： %d" % len(self.searchFilmList), end='')
continue
except:
# await page.close()
# await browser.close()
# page = None
# browser = None
# self.del_proxy(proxy)
print('检测到有异常')
# continue
root = await page.J('#wrapper')
# root_text = await page.evaluate('item=>item.textContent', root)
item_root = await page.J('.item-root')
while not item_root:
await asyncio.sleep(1)
# 没有搜到
if not item_root:
data = {'douban_url': 'no', 'film_name': film_name}
# Db.update_dict('film_list', {'id': film['id']}, data)
print('no find')
continue
# print(await page.evaluate('item=>item.textContent', item_root))
# print(await page.evaluate('item=>item.src', await item_root.J('.cover-link >.cover')))
# print(await page.evaluate('item=>item.href', await item_root.J('.cover-link')))
title = await page.evaluate('item=>item.textContent', await item_root.J('.title-text'))
douban_rating = await item_root.J('.rating >.rating_nums')
if douban_rating:
douban_rating = await page.evaluate('item=>item.textContent', douban_rating)
else:
douban_rating = 1.1
data = {
'douban_pic': await page.evaluate('item=>item.src', await item_root.J('.cover-link >.cover')),
'douban_url': await page.evaluate('item=>item.href', await item_root.J('.cover-link')),
'douban_rating': douban_rating,
'film_name': film_name
}
self.film_coll.update_one({'_id': film['_id']}, {'$set': data})
print("\r剩余： %d" % len(self.searchFilmList), end='')
if page:
await page.close()
if browser:
await browser.close()
page = None
browser = None
async def get_film_info(self):
try:
film = self.searchFilmList.pop()
except:
return
proxy = self.get_proxy()
headers = {
'Host': 'movie.douban.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3904.108 Safari/537.36'}
# 实例化好一个请求对象
async with aiohttp.ClientSession(trust_env=True) as sess:
# 调用get发起请求，返回一个响应对象
async with sess.get(url=film['douban_url'], headers=headers, timeout=5000,
proxy="http://" + proxy) as response:
# await asyncio.sleep(random.uniform(0.5, 1.5))
if response.status != 200:
self.del_proxy(proxy)
print('err')
return
# pool.submit(self.get_film_info(Db, pool))
page_text = await response.text()
soup = BeautifulSoup(page_text, 'html.parser')
info = soup.find('div', id="info")
if info.find('span', text=re.compile("^职业.?")) or info.find('span', text=re.compile("^性别.?")) or \
info.find('dt', text=re.compile("^官方网站.?")) or info.find('h2', text=re.compile("影人简介.?")):
self.film_coll.update_one({'_id': film['_id']}, {'$set': {'douban_state': 1}})
self.count -= 1
print("\r剩余-： %d" % self.count, end='')
return
producer = info.find(name='span', class_=re.compile('pl'), text=re.compile("^制片国家.?"))
producer = producer.next_sibling.replace(' ', '') if producer is not None else ''
if producer == '':
producer1 = info.find('span', attrs={'property': 'v:initialReleaseDate'})
producer1 = producer1.string if producer1 else ''
producer1 = re.findall(r'[0123456789-]?\((.*)\)', producer1) if producer1 else []
producer = producer1[0] if len(producer1) > 0 else ''
data_str = soup.find('script', attrs={'type': 'application/ld+json'}).string
data_dict = json.loads(data_str, strict=False)
douban_name = data_dict['name']
director = data_dict['director'][0]['name'] if len(data_dict['director']) > 0 else ''
actor = []
for x in data_dict['actor']:
actor.append(x['name'])
# year = fi.lm['year'] if film['year'] and film['year'] != -1 else '1990'
# initial_date = data_dict['datePublished'] if data_dict['datePublished'] != '' else str(year) + '-01-01'
# douban_initial_year = initial_date.split('-')[0] if initial_date else year
year = 1990
if data_dict['datePublished']:
initial_date = data_dict['datePublished']
else:
if film['year'] and 1960 < int(film['year']) < 2022:
year = film['year']
initial_date = str(year) + '-01-01'
douban_initial_year = initial_date.split('-')[0] if initial_date else year
if len(soup.select('#link-report span')) > 0:
if soup.select('#link-report .all'):
des_text = soup.select('#link-report .all')[0].get_text()
else:
des_text = soup.select('#link-report span')[0].get_text()
des = self.filter_film_name(des_text)
else:
des = ''
data = {
'douban_genre': data_dict['genre'],
'douban_country': producer,
'douban_initial_date': initial_date,
'douban_initial_year': douban_initial_year,
'actor': actor,
'director': director,
'douban_name': douban_name if len(douban_name) <= 200 else douban_name[:100],
'douban_des': self.filter_film_name(des)
}
print(data)
print(film)
# Db.update_byid('film_list', data)
# Db.update_dict('film_info', {'film_id': film['id']}, data2)
self.film_coll.update_one({'_id': film['_id']}, {'$set': data})
self.count -= 1
print("\r剩余： %d" % self.count, end='')
import re
from flask import Flask, render_template, request, jsonify
from flask_cors import CORS
from bson.objectid import ObjectId
import site_config
from utils.MongoUtil import MongoUtil
app = Flask(__name__)
# 实现跨域访问
cors = CORS(app, resources={r"*": {"origins": "*"}})
Mg = MongoUtil()
film_filters = [{
'key':
'classify',
'list': [{
'name': '电影',
'value': 1
}, {
'name': '连续剧',
'value': 2
}, {
'name': '动漫',
'value': 3
}, {
'name': '综艺',
'value': 5
}]
}, {
'key': 'order',