def match_douban_data(request): ret = {'status': True, 'error': None, 'data': None} try: type_list = ['movie', 'tv'] tag_list = { 'movie': [ '热门', '最新', '经典', '华语', '欧美', '韩国', '日本', '动作', '喜剧', '爱情', '科幻', '悬疑', '恐怖', '治愈' ], 'tv': ['热门', '美剧', '英剧', '日剧', '国产剧', '港剧', '日本动画', '综艺', '纪录片'] } for video_type in type_list: admin_flag = DBHandler.redis_load('admin_flag') if admin_flag == b'0': print('管理员终止') break for tag in tag_list[video_type]: admin_flag = DBHandler.redis_load('admin_flag') if admin_flag == b'0': print('管理员终止') break filename = 'douban/%s_%s.json' % (video_type, tag) with open(filename, 'r', encoding='utf-8') as f: detail_list = json.load(f) for item in detail_list['subjects']: GetPageData.match_score_by_name(item) except Exception as e: print(e) ret['status'] = False ret['error'] = e finally: return HttpResponse(json.dumps(ret))
def update_rating(request): res_status = False try: print('————————开始更新豆瓣评分数据————————') data_list = DBHandler.get_none_rating() # 获取所有电影 for item in data_list: # 判断管理员指令 admin_flag = DBHandler.redis_load('admin_flag') if admin_flag == b'0': print('管理员终止') break douban_id = item['vod_douban_id'] rating = GetPageData.getRating(douban_id) if rating: DBHandler.dump_rating(item['vod_id'], rating) print(item['vod_name'], rating, '保存成功[rating]') res_status = True except Exception as e: print('update_rating exception:', e) print(traceback.print_exc()) res_status = 'update_rating exception:%s' % e finally: print('————————豆瓣评分数据更新完毕————————') return HttpResponse(res_status)
def get_douban_rating(request): res_status = False flag = True timeout = 0 try: print('————————开始获取豆瓣id数据————————') # data_list = DBHandler.load_type_data( # **{'vod_cid': '1', 'no_douban_id': True}) # 获取所有电影 data_list = DBHandler.load_type_data(**{ 'vod_cid': '2', 'no_douban_id': True }) # 获取所有电视剧 none_list = DBHandler.redis_loadlist('douban_none_list') none_list2 = DBHandler.redis_loadlist('douban_none_list2') print('无豆瓣id的视频总数:%s' % len(data_list)) print('无匹配视频总数:%s' % len(none_list)) print('重复数据的视频总数:%s' % len(none_list2)) # while flag: # print(bytes('75519',encoding='utf-8') not in none_list and bytes('75519',encoding='utf-8') not in none_list2) # return HttpResponse(none_list) for item in data_list: admin_flag = DBHandler.redis_load('admin_flag') if admin_flag == b'0': print('管理员终止') break b_id = bytes(item['vod_id'], encoding='utf-8') # 不存在豆瓣id,则查找id if item['vod_douban_id'] is None and (b_id not in none_list and b_id not in none_list2): # print(item['vod_douban_id'],item['vod_rating']) i = 0 while i < 2: douban_id = GetPageData.findID(item['vod_name'], item['vod_year']) if douban_id: # 如果查找到豆瓣id timeout = 0 if type(douban_id) == list: print(item['vod_name'], '匹配到多个豆瓣id数据,存入列表2', douban_id) if b_id not in none_list2: none_list2.append(b_id) else: DBHandler.dump_douban_id(item['vod_id'], douban_id) print(item['vod_name'], douban_id, end=' ') rating = GetPageData.getRating(douban_id) if rating: print(rating, end='') DBHandler.dump_rating(item['vod_id'], rating) print('保存成功[douban]') # 防止账号被封,随机延迟 # time.sleep(1 + float(random.randint(40, 100)) / 20) time.sleep(0.5) break else: # 未查找到豆瓣id i += 1 # if i ==1: # print(item['vod_name']) continue # test_net = GetPageData.findID('爱情公寓2', '2011') # if test_net is None: # 网络异常 # print('----------地址被限制,稍后重试----------') # if none_list: # DBHandler.redis_dumplist( # 'douban_none_list', none_list) # none_list = [] # print('none_list 暂存') # if none_list2: # DBHandler.redis_dumplist( # 'douban_none_list2', none_list2) # none_list2 = [] # print('none_list2 暂存') # # time.sleep(180) # timeout += 1 # # if timeout == 5: # # print('连续五次失败,等待30分钟') # # time.sleep(1800) # # print('----------重新启动查找------------') # # continue # # elif timeout >= 10: # # print('-----连续十次失败,等待1小时-----') # # time.sleep(3600) # # timeout = 0 # # continue # else: # print(item['vod_name'], '无豆瓣id数据,存入列表') # if b_id not in none_list: # none_list.append(b_id) res_status = True except Exception as e: print('get_douban_rating exception:', e) print(traceback.print_exc()) res_status = 'get_douban_rating exception:%s' % e finally: if none_list: DBHandler.redis_dumplist('douban_none_list', none_list) if none_list2: DBHandler.redis_dumplist('douban_none_list2', none_list2) DBHandler.redis_dump('admin_flag', '1') print('————————豆瓣id获取完毕————————') return HttpResponse(res_status)