def crawler(): browser = Chrome(cache_path=r"E:\temp") account_list = Utils.io.load_string("huya_account_list.txt") for account_url in account_list.split("\n"): browser.get(account_url) # 读取直播间订阅数量 text_subscribe = "" try: label_subscribe = browser.find_element_by_xpath( '//*[@id="activityCount"]') if label_subscribe is not None: text_subscribe = label_subscribe.text except NoSuchElementException: pass # 读取直播间ID text_id = "" try: label_id = browser.find_element_by_css_selector( '#J_roomHeader > div.room-hd-l > div.host-info > div.host-detail.J_roomHdDetail > span.host-rid' ) if label_id is not None: text_id = label_id.text except NoSuchElementException: pass print(account_url, text_id, text_subscribe) time.sleep(3)
def running(self): driver = Chrome(cache_path=r"E:\temp") driver.get("https://cc.julive.com/project/s") bs = BeautifulSoup(driver.page_source, 'lxml') # 将网页转化为BeautifulSoup结构 city_dict = dict() for element_city in bs.select( "body > div.container-5-2.container > div.header-v5.header-v5-2.header-normal > div > div.inn-p > div.city-position.city-tip > div.city-change-list-new > ul > li > ul > li> a" ): city_name = element_city.text city_url = element_city["href"] city_dict[city_name] = city_url return city_dict
def crawler(): browser = Chrome(cache_path=r"E:\temp") browser.get("https://cc.julive.com/project/s") bs = BeautifulSoup(browser.page_source, 'lxml') # 将网页转化为BeautifulSoup结构 city_dict = dict() for element_city in bs.select( "body > div.container-5-2.container > div.header-v5.header-v5-2.header-normal > div > div.inn-p > div.city-position.city-tip > div.city-change-list-new > ul > li > ul > li> a" ): city_name = element_city.text city_url = element_city["href"] city_dict[city_name] = city_url print(city_name, city_url) Utils.io.write_json("julive_city_url_20191217.json", city_dict)
def running(self): driver = Chrome(cache_path=r"E:\temp") # ChromeDriver可执行文件的路径 driver.get("http://piaofang.maoyan.com/dashboard/web-heat") time.sleep(1) res = [] for movie_label in driver.find_elements_by_css_selector( "#app > div > div > div.dashboard-content > div.dashboard-list.dashboard-left.bg > div.movielist-container > div > table > tbody > tr" ): res.append([ movie_label.find_element_by_class_name("moviename-index").text, movie_label.find_element_by_class_name("moviename-name").text, movie_label.find_element_by_class_name("moviename-info").text, movie_label.find_element_by_class_name("heat-text").text, movie_label.find_element_by_class_name("last-col").text ]) return res
def crawler(file_name): """ LeetCode题目列表爬虫 :param file_name: 抓取结果存储文件地址 """ selenium = Chrome(cache_path=r"E:\Temp") # 启动Chrome浏览器驱动 selenium.get(PROBLEMS_SET_URL) # 打开题库页面 # 获取题目列表(Json格式) problems_all_json = requests.get(PROBLEMS_ALL_API).json() print("解析题目总数:", problems_all_json["num_total"]) # 解析题目列表(生成problem实例列表):key=题目ID,value=题目的problem实例 result_problems = {} for problem in problems_all_json["stat_status_pairs"]: problem_elem = Problem(problem) result_problems[problem_elem.id] = problem_elem # 获取题目标签(Json格式)并将结果写入到题目列表中 problems_tags_json = requests.get(PROBLEMS_TAGS_API).json() for topic in problems_tags_json["topics"]: tag_name = topic["translatedName"] if topic[ "translatedName"] else topic["name"] for qid in topic["questions"]: if qid in result_problems: result_problems[qid].add_tag(tag_name) else: print("题目ID未找到:", qid, tag_name) # 获取题目翻译(Json格式)并将结果写入到题目列表中 translations_json = selenium.post(GRAPHQL_API, json.dumps(GRAPHQL_QUERY_TRANSLATIONS), payload=True) for problem in translations_json["data"]["translations"]: if (qid := int(problem["questionId"])) in result_problems: result_problems[qid].title = problem["title"]
def crawler(live_list_path): driver = Chrome(cache_path=r"E:\Temp") account_list = Utils.io.load_string(live_list_path) for account_url in account_list.split("\n"): driver.get(account_url) time.sleep(3) text_subscribe = "" for _ in range(10): try: label_subscribe = driver.find_element_by_xpath( '//*[@id="js-player-title"]/div/div[4]/div/span') if label_subscribe.text is not None and label_subscribe.text != "": text_subscribe = label_subscribe.text break time.sleep(1) except NoSuchElementException: time.sleep(1) print(account_url, text_subscribe)
def crawler(live_name, live_url, mysql): browser = Chrome(cache_path=r"E:\temp") browser.get(live_url) # 访问目标虎牙主播的直播间 time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time())) table_name = "huya_{}".format(time_string) sql_create = "CREATE TABLE live_barrage.`huya_{}` (" \ "`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \ "`type` char(10) DEFAULT NULL COMMENT '弹幕类型'," \ "`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \ " `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \ " `user_noble` int(11) DEFAULT NULL COMMENT '弹幕发布者贵族等级'," \ " `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \ " `gift_name` varchar(40) DEFAULT NULL COMMENT '赠送礼物名称'," \ " `gift_num` int(11) DEFAULT '0' COMMENT '赠送礼物数量'," \ " `other` varchar(60) DEFAULT NULL COMMENT '弹幕其他信息'," \ " PRIMARY KEY (`bid`)" \ ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='虎牙弹幕({})';" mysql.create(sql_create.format(time_string, live_name)) print("开始抓取虎牙直播弹幕.....") total_time = 0 total_num = 0 data_id_max = 0 for num in range(int(36000 / 0.5)): start_time = time.time() label_html = browser.find_element_by_id( "chat-room__list").get_attribute("innerHTML") bs = BeautifulSoup(label_html, 'lxml') # 将网页内容解析为Soup对象 barrage_list = [] for label in bs.select("li"): data_id = int(label["data-id"]) # 提取:弹幕ID if data_id <= data_id_max: # 依据弹幕的ID判断弹幕是否还未抓取 if data_id > data_id_max - 101: continue data_id_max = data_id barrage_info = { "bid": data_id, # 弹幕ID "type": "", # 弹幕所属类型 "user_name": "", # 弹幕发布者名称 "user_noble": 0, # 弹幕发布者贵族等级 "content": "", # 弹幕内容 "gift_name": "", # 礼物名称 "gift_num": 0, # 礼物数量 "other": "" # 其他信息 } category = str(label.select_one("li > div")["class"]) # 提取:弹幕类型 if "msg-smog" in category: # 处理smog类型弹幕(普通弹幕) barrage_info["type"] = "SG" barrage_info["user_name"] = label.select_one( "li > div > span:nth-child(1)").text barrage_info["content"] = label.select_one( "li > div > span:nth-child(3)").text elif "msg-normal" in category: # 处理普通类型弹幕(普通弹幕) barrage_info["type"] = "NM" barrage_info["user_name"] = label.select_one( "li > div > span:nth-child(2)").text barrage_info["content"] = label.select_one( "li > div > span:nth-child(5)").text elif "msg-nobleEnter" in category: # 处理nobleEnter类型弹幕(贵族进入弹幕) barrage_info["type"] = "NE" barrage_info["user_name"] = label.select_one( "li > div > div > p > span:nth-child(1)").text barrage_info["user_noble"] = label.select_one( "li > div > div")["class"] barrage_info["content"] = "驾临直播间" elif "msg-nobleSpeak" in category: # 处理nobleSpeak类型弹幕(贵族发言) barrage_info["type"] = "NS" barrage_info["user_name"] = label.select_one( "li > div > p > span:nth-child(2)").text barrage_info["user_noble"] = int( label.select_one("li > div")["class"]) barrage_info["content"] = label.select_one( "li > div > p > span:nth-child(5)").text elif "tit-h-send" in category: # 处理send类型提示(礼物赠送提示) barrage_info["type"] = "SD" barrage_info["user_name"] = label.select_one( "li > div > span:nth-child(1)").text barrage_info["gift_name"] = label.select_one( "li > div > span:nth-child(3) > img")["alt"] barrage_info["gift_num"] = int( label.select_one( "li > div > span:nth-child(4) > img").text) elif "msg-onTVLottery" in category: barrage_info["type"] = "TV" barrage_info["user_name"] = label.select_one( "li > div > span:nth-child(2)").text barrage_info["content"] = label.select_one( "li > div > div > span").text elif "msg-auditorSys" in category: # 处理msg-auditorSys类型提示(系统提示) barrage_info["type"] = "AS" barrage_info["other"] = label.text elif "msg-sys" in category: # 处理msg-sys类型提示(系统提示) barrage_info["type"] = "SY" barrage_info["other"] = label.text else: # 处理其他类型 barrage_info.update(type="OT", other="弹幕名称" + category) barrage_list.append(barrage_info) mysql.insert(table_name, barrage_list) total_num += 1 total_time += 1000 * (time.time() - start_time) wait_time = 0.5 if wait_time > (time.time() - start_time): time.sleep(0.5 - (time.time() - start_time)) print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max, ")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(", round(total_time), "/", total_num, ")")
def crawler(live_name, live_url, mysql): browser = Chrome(cache_path=r"E:\Temp") # 打开Chrome浏览器 browser.get(live_url) # 访问目标斗鱼主播的直播间 time.sleep(10) time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time())) table_name = "douyu_{}".format(time_string) sql_create = "CREATE TABLE live_barrage.`douyu_{}` (" \ "`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \ "`type` varchar(60) DEFAULT NULL COMMENT '弹幕类型'," \ "`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \ " `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \ " `user_level` int(11) DEFAULT NULL COMMENT '弹幕发布者等级'," \ " `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \ " `text` varchar(100) DEFAULT NULL COMMENT '弹幕其他信息'," \ " PRIMARY KEY (`bid`)" \ ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='斗鱼弹幕({})';" mysql.create(sql_create.format(time_string, live_name)) print("开始抓取斗鱼直播弹幕.....") total_time = 0 total_num = 0 # screenshot = 0 barrage_id_list = list() data_id_max = 0 for num in range(int(36000 / 0.5)): start_time = time.time() label_html = browser.find_element_by_id( "js-barrage-list").get_attribute("innerHTML") soup = BeautifulSoup(label_html, 'lxml') # 将网页内容解析为Soup对象 barrage_list = [] for label in soup.select("li"): bid = str(label["id"]) # 提取:弹幕ID if bid in barrage_id_list: continue barrage_id_list.append(bid) if len(barrage_id_list) > 200: barrage_id_list.remove(barrage_id_list[0]) barrage_info = { "type": "", # 弹幕所属类型 "user_name": "", # 弹幕发布者名称 "user_level": 0, # 弹幕发布者等级 "content": "", # 弹幕内容 "text": "" # 其他信息 } type_class = label.select_one("li > div")["class"] if "Barrage-notice" in type_class and "normalBarrage" not in type_class: barrage_info["type"] = "NOTICE" elif "normalBarrage" in type_class: barrage_info["type"] = "NORMAL" elif "Barrage-userEnter" in type_class: barrage_info["type"] = "ENTER" elif "Barrage-message" in type_class: barrage_info["type"] = "MESSAGE" for info_label in label.select("li > div > span"): info_label_class = info_label["class"] if "UserLevel" in info_label_class: barrage_info["user_level"] = re.search( "[0-9]+", info_label["title"]).group() elif "Barrage-nickName" in info_label_class: barrage_info["user_name"] = info_label.text.replace( " ", "") elif "Barrage-content" in info_label_class: barrage_info["content"] = info_label.text.replace(" ", "") elif "Barrage-text" in info_label_class: barrage_info["text"] = info_label.text.replace(" ", "") barrage_list.append(barrage_info) if len(barrage_list) < 200: mysql.insert(table_name, barrage_list) total_num += 1 total_time += 1000 * (time.time() - start_time) print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max, ")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(", round(total_time), "/", total_num, ")") else: total_num += 1 total_time += 1000 * (time.time() - start_time) print("本次时间范围内弹幕列表未自动向下滚动...") wait_time = 0.5 if wait_time > (time.time() - start_time): time.sleep(0.5 - (time.time() - start_time)) data_id_max += len(barrage_list)
""" 猫眼网播热度采集 需要第三方模块: Selenium4R >= 0.0.3 @author: ChangXing @version: 1.0 @create: 2020.05.26 @revise: - """ import time from Selenium4R import Chrome if __name__ == "__main__": browser = Chrome(cache_path=r"E:\temp") # ChromeDriver可执行文件的路径 browser.get("http://piaofang.maoyan.com/dashboard/web-heat") time.sleep(1) for movie_label in browser.find_elements_by_css_selector( "#app > div > div > div.dashboard-content > div.dashboard-list.dashboard-left.bg > div.movielist-container > div > table > tbody > tr"): print("排名:", movie_label.find_element_by_class_name("moviename-index").text) print("名称:", movie_label.find_element_by_class_name("moviename-name").text) print("信息:", movie_label.find_element_by_class_name("moviename-info").text) print("信息:", movie_label.find_element_by_class_name("heat-text").text) print("信息:", movie_label.find_element_by_class_name("last-col").text)