def crawler_city_resources():
"""
采集城市房源数量
"""
cities = tool.file.load_as_json("anjuke_city_code.json")
city_infor = tool.file.load_as_json("anjuke_city_infor.json")
browser = tool.open_chrome(use_user_dir=False)
for city_name in cities:
if city_name not in city_infor:
city_code = cities[city_name]
browser.get("https://" + city_code +
".fang.anjuke.com/?from=navigation")
bs = BeautifulSoup(browser.page_source,
'lxml') # 将网页转化为BeautifulSoup结构
city_label = bs.select_one(
"#container > div.list-contents > div.list-results > div.key-sort > div.sort-condi > span > em"
)
if city_label is not None:
city_num = city_label.text
else:
city_num = 0
if bs.select_one(
"#header > div.site-search > div.sel-city > a > span"
) is None:
print("可能出现反爬虫，请处理...")
time.sleep(300)
break
city_num = int(city_num)
city_infor[city_name] = city_num
tool.file.write_json("anjuke_city_infor.json", city_infor)
time.sleep(2)
def crawler():
browser = tool.open_chrome(use_user_dir=False)
account_list = tool.file.load_as_string("huya_account_list.txt")
for account_url in account_list.split("\n"):
browser.get(account_url)
# 读取直播间订阅数量
text_subscribe = ""
try:
label_subscribe = browser.find_element_by_xpath(
'//*[@id="activityCount"]')
if label_subscribe is not None:
text_subscribe = label_subscribe.text
except NoSuchElementException:
pass
# 读取直播间ID
text_id = ""
try:
label_id = browser.find_element_by_css_selector(
'#J_roomHeader > div.room-hd-l > div.host-info > div.host-detail.J_roomHdDetail > span.host-rid'
)
if label_id is not None:
text_id = label_id.text
except NoSuchElementException:
pass
print(account_url, text_id, text_subscribe)
time.sleep(3)
def crawler():
browser = tool.open_chrome(use_user_dir=False)
browser.get("https://cc.julive.com/project/s")
bs = BeautifulSoup(browser.page_source, 'lxml') # 将网页转化为BeautifulSoup结构
city_dict = dict()
for element_city in bs.select(
"body > div.container-5-2.container > div.header-v5.header-v5-2.header-normal > div > div.inn-p > div.city-position.city-tip > div.city-change-list-new > ul > li > ul > li> a"
):
city_name = element_city.text
city_url = element_city["href"]
city_dict[city_name] = city_url
print(city_name, city_url)
tool.file.write_json("julive_city_url_20191217.json", city_dict)
def crawler_city_list():
"""
采集城市编码列表
"""
browser = tool.open_chrome(use_user_dir=False)
browser.get("https://www.anjuke.com/sy-city.html")
bs = BeautifulSoup(browser.page_source, 'lxml') # 将网页转化为BeautifulSoup结构
city_dict = dict()
for city_label in bs.select(
"body > div.content > div > div.letter_city > ul > li > div > a"):
city_name = city_label.get_text()
city_code = city_label["href"].replace("https://",
"").replace(".anjuke.com", "")
city_dict[city_name] = city_code
print(city_name, city_code)
tool.file.write_json("anjuke_city_code.json", city_dict)
def crawler(live_list_path):
browser = tool.open_chrome(use_user_dir=False)
account_list = tool.file.load_as_string(live_list_path)
for account_url in account_list.split("\n"):
browser.get(account_url)
time.sleep(3)
text_subscribe = ""
for _ in range(10):
try:
label_subscribe = browser.find_element_by_xpath(
'//*[@id="js-player-title"]/div/div[4]/div/span')
if label_subscribe.text is not None and label_subscribe.text != "":
text_subscribe = label_subscribe.text
break
time.sleep(1)
except NoSuchElementException:
time.sleep(1)
print(account_url, text_subscribe)
tweet_info["retweets"] = tweet_retweets
tweet_info["likes"] = tweet_likes
tweet_list.append(tweet_info)
# 向下滚动到最下面的一条推文
if last_label_tweet is not None:
driver.execute_script("arguments[0].scrollIntoView();", last_label_tweet) # 滑动到推文标签
time.sleep(1)
else:
break
return tweet_list
if __name__ == "__main__":
selenium = tool.open_chrome() # 打开Selenium控制的Chrome浏览器
mySQL = tool.mysql_connect("Huabang") # 构造MySQL数据库连接对象
if "Huabang" in env.DATA and "Media List" in env.DATA["Huabang"]:
for media_item in env.DATA["Huabang"]["Media List"]:
# if media_item[0] < 440:
# continue
print("开始抓取媒体:", media_item[1], "(", media_item[0], ")", "-", media_item[3], "(", media_item[2], ")")
tweet_template = {
"media_id": media_item[0],
"media_name": media_item[1],
"tweet_id": None,
"is_retweet": 0,
"time": None,
"text": None,
"replies": None,
def crawler(live_name, live_url, mysql):
browser = tool.open_chrome(use_user_dir=False) # 打开Chrome浏览器
browser.get(live_url) # 访问目标Bilibili主播的直播间
time.sleep(10)
time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))
table_name = "bilibili_{}".format(time_string)
sql_create = "CREATE TABLE live_barrage.`bilibili_{}` (" \
"`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
"`type` varchar(60) DEFAULT NULL COMMENT '弹幕类型'," \
"`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
" `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
" `user_id` int(11) DEFAULT NULL COMMENT '弹幕发布者等级'," \
" `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
" PRIMARY KEY (`bid`)" \
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Bilibili弹幕({})';"
mysql.create(sql_create.format(time_string, live_name))
print("开始抓取Bilibili直播弹幕.....")
total_time = 0
total_num = 0
update_time = 0
data_id_max = 0
for num in range(int(36000 / 0.5)):
start_time = time.time()
label_html = browser.find_element_by_id(
"chat-history-list").get_attribute("outerHTML")
soup = BeautifulSoup(label_html, 'lxml') # 将网页内容解析为Soup对象
barrage_list = []
for label in soup.select("#chat-history-list > div"):
barrage_info = {
"type": "", # 弹幕所属类型
"user_name": "", # 弹幕发布者名称
"user_id": 0, # 弹幕发布者等级
"content": "", # 弹幕内容
}
type_class = label["class"]
if "danmaku-item" in type_class:
barrage_info["type"] = "NORMAL"
elif "welcome-msg" in type_class:
barrage_info["type"] = "ENTER"
elif "system-msg" in type_class:
barrage_info["type"] = "MESSAGE"
elif "gift-item" in type_class:
barrage_info["type"] = "GIFT"
if barrage_info.get("type") == "NORMAL":
temp_time = int(label["data-ts"])
if temp_time >= update_time:
update_time = temp_time
barrage_info["user_name"] = label["data-uname"]
barrage_info["user_id"] = label["data-uid"]
barrage_info["content"] = label["data-danmaku"]
barrage_list.append(barrage_info)
mysql.insert_pure(table_name, barrage_list)
total_num += 1
total_time += 1000 * (time.time() - start_time)
wait_time = 0.5
if wait_time > (time.time() - start_time):
time.sleep(0.5 - (time.time() - start_time))
data_id_max += len(barrage_list)
print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
round(total_time), "/", total_num, ")")
following_count)
template["following_count"] = following_count
if abs(template["followers_count"] - followers_count) > 1000 \
or (abs(template["followers_count"] - followers_count) > 0 and template["followers_count"] < 10000):
print("修正关注者数量:", template["followers_count"], "→",
followers_count)
template["followers_count"] = followers_count
except TypeError:
print("修改正在关注、关注者数量失败，最终取值：正在关注 =", template["following_count"],
"、关注数 =", template["followers_count"])
return template
if __name__ == "__main__":
selenium = tool.open_chrome(use_user_dir=False) # 打开Selenium控制的Chrome浏览器
mySQL = tool.mysql_connect("Huabang") # 构造MySQL数据库连接对象
if "Huabang" in env.DATA and "Media List" in env.DATA["Huabang"]:
for media_item in env.DATA["Huabang"]["Media List"]:
if media_item[0] < 133:
continue
print("开始抓取媒体:", media_item[1], "(", media_item[0], ")", "-",
media_item[3], "(", media_item[2], ")")
user_template = {
"media_id": media_item[0],
"media_name": media_item[1],
"name": None,
"username": None,
"birthday": None,
"biography": None,
def crawler(live_name, live_url, mysql):
browser = tool.open_chrome(use_user_dir=False)
browser.get(live_url) # 访问目标虎牙主播的直播间
time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))
table_name = "huya_{}".format(time_string)
sql_create = "CREATE TABLE live_barrage.`huya_{}` (" \
"`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
"`type` char(10) DEFAULT NULL COMMENT '弹幕类型'," \
"`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
" `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
" `user_noble` int(11) DEFAULT NULL COMMENT '弹幕发布者贵族等级'," \
" `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
" `gift_name` varchar(40) DEFAULT NULL COMMENT '赠送礼物名称'," \
" `gift_num` int(11) DEFAULT '0' COMMENT '赠送礼物数量'," \
" `other` varchar(60) DEFAULT NULL COMMENT '弹幕其他信息'," \
" PRIMARY KEY (`bid`)" \
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='虎牙弹幕({})';"
mysql.create(sql_create.format(time_string, live_name))
print("开始抓取虎牙直播弹幕.....")
total_time = 0
total_num = 0
data_id_max = 0
for num in range(int(36000 / 0.5)):
start_time = time.time()
label_html = browser.find_element_by_id(
"chat-room__list").get_attribute("innerHTML")
bs = BeautifulSoup(label_html, 'lxml') # 将网页内容解析为Soup对象
barrage_list = []
for label in bs.select("li"):
data_id = int(label["data-id"]) # 提取:弹幕ID
if data_id <= data_id_max: # 依据弹幕的ID判断弹幕是否还未抓取
if data_id > data_id_max - 101:
continue
data_id_max = data_id
barrage_info = {
"bid": data_id, # 弹幕ID
"type": "", # 弹幕所属类型
"user_name": "", # 弹幕发布者名称
"user_noble": 0, # 弹幕发布者贵族等级
"content": "", # 弹幕内容
"gift_name": "", # 礼物名称
"gift_num": 0, # 礼物数量
"other": "" # 其他信息
}
category = str(label.select_one("li > div")["class"]) # 提取:弹幕类型
if "msg-smog" in category: # 处理smog类型弹幕(普通弹幕)
barrage_info["type"] = "SG"
barrage_info["user_name"] = label.select_one(
"li > div > span:nth-child(1)").text
barrage_info["content"] = label.select_one(
"li > div > span:nth-child(3)").text
elif "msg-normal" in category: # 处理普通类型弹幕(普通弹幕)
barrage_info["type"] = "NM"
barrage_info["user_name"] = label.select_one(
"li > div > span:nth-child(2)").text
barrage_info["content"] = label.select_one(
"li > div > span:nth-child(5)").text
elif "msg-nobleEnter" in category: # 处理nobleEnter类型弹幕(贵族进入弹幕)
barrage_info["type"] = "NE"
barrage_info["user_name"] = label.select_one(
"li > div > div > p > span:nth-child(1)").text
barrage_info["user_noble"] = label.select_one(
"li > div > div")["class"]
barrage_info["content"] = "驾临直播间"
elif "msg-nobleSpeak" in category: # 处理nobleSpeak类型弹幕(贵族发言)
barrage_info["type"] = "NS"
barrage_info["user_name"] = label.select_one(
"li > div > p > span:nth-child(2)").text
barrage_info["user_noble"] = int(
label.select_one("li > div")["class"])
barrage_info["content"] = label.select_one(
"li > div > p > span:nth-child(5)").text
elif "tit-h-send" in category: # 处理send类型提示(礼物赠送提示)
barrage_info["type"] = "SD"
barrage_info["user_name"] = label.select_one(
"li > div > span:nth-child(1)").text
barrage_info["gift_name"] = label.select_one(
"li > div > span:nth-child(3) > img")["alt"]
barrage_info["gift_num"] = int(
label.select_one(
"li > div > span:nth-child(4) > img").text)
elif "msg-onTVLottery" in category:
barrage_info["type"] = "TV"
barrage_info["user_name"] = label.select_one(
"li > div > span:nth-child(2)").text
barrage_info["content"] = label.select_one(
"li > div > div > span").text
elif "msg-auditorSys" in category: # 处理msg-auditorSys类型提示(系统提示)
barrage_info["type"] = "AS"
barrage_info["other"] = label.text
elif "msg-sys" in category: # 处理msg-sys类型提示(系统提示)
barrage_info["type"] = "SY"
barrage_info["other"] = label.text
else: # 处理其他类型
barrage_info.update(type="OT", other="弹幕名称" + category)
barrage_list.append(barrage_info)
mysql.insert(table_name, barrage_list)
total_num += 1
total_time += 1000 * (time.time() - start_time)
wait_time = 0.5
if wait_time > (time.time() - start_time):
time.sleep(0.5 - (time.time() - start_time))
print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
round(total_time), "/", total_num, ")")
def crawler(live_name, live_url, mysql):
browser = tool.open_chrome(use_user_dir=False) # 打开Chrome浏览器
browser.get(live_url) # 访问目标斗鱼主播的直播间
time.sleep(10)
time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))
table_name = "douyu_{}".format(time_string)
sql_create = "CREATE TABLE live_barrage.`douyu_{}` (" \
"`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
"`type` varchar(60) DEFAULT NULL COMMENT '弹幕类型'," \
"`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
" `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
" `user_level` int(11) DEFAULT NULL COMMENT '弹幕发布者等级'," \
" `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
" `text` varchar(100) DEFAULT NULL COMMENT '弹幕其他信息'," \
" PRIMARY KEY (`bid`)" \
") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='斗鱼弹幕({})';"
mysql.create(sql_create.format(time_string, live_name))
print("开始抓取斗鱼直播弹幕.....")
total_time = 0
total_num = 0
# screenshot = 0
barrage_id_list = list()
data_id_max = 0
for num in range(int(36000 / 0.5)):
start_time = time.time()
label_html = browser.find_element_by_id(
"js-barrage-list").get_attribute("innerHTML")
soup = BeautifulSoup(label_html, 'lxml') # 将网页内容解析为Soup对象
barrage_list = []
for label in soup.select("li"):
bid = str(label["id"]) # 提取:弹幕ID
if bid in barrage_id_list:
continue
barrage_id_list.append(bid)
if len(barrage_id_list) > 200:
barrage_id_list.remove(barrage_id_list[0])
barrage_info = {
"type": "", # 弹幕所属类型
"user_name": "", # 弹幕发布者名称
"user_level": 0, # 弹幕发布者等级
"content": "", # 弹幕内容
"text": "" # 其他信息
}
type_class = label.select_one("li > div")["class"]
if "Barrage-notice" in type_class and "normalBarrage" not in type_class:
barrage_info["type"] = "NOTICE"
elif "normalBarrage" in type_class:
barrage_info["type"] = "NORMAL"
elif "Barrage-userEnter" in type_class:
barrage_info["type"] = "ENTER"
elif "Barrage-message" in type_class:
barrage_info["type"] = "MESSAGE"
for info_label in label.select("li > div > span"):
info_label_class = info_label["class"]
if "UserLevel" in info_label_class:
barrage_info["user_level"] = re.search(
"[0-9]+", info_label["title"]).group()
elif "Barrage-nickName" in info_label_class:
barrage_info["user_name"] = info_label.text.replace(
" ", "")
elif "Barrage-content" in info_label_class:
barrage_info["content"] = info_label.text.replace(" ", "")
elif "Barrage-text" in info_label_class:
barrage_info["text"] = info_label.text.replace(" ", "")
barrage_list.append(barrage_info)
if len(barrage_list) < 200:
mysql.insert(table_name, barrage_list)
total_num += 1
total_time += 1000 * (time.time() - start_time)
print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
round(total_time), "/", total_num, ")")
else:
total_num += 1
total_time += 1000 * (time.time() - start_time)
print("本次时间范围内弹幕列表未自动向下滚动...")
wait_time = 0.5
if wait_time > (time.time() - start_time):
time.sleep(0.5 - (time.time() - start_time))
data_id_max += len(barrage_list)