Пример #1
0
def crawler_city_resources():
    """
    采集城市房源数量
    """
    cities = tool.file.load_as_json("anjuke_city_code.json")
    city_infor = tool.file.load_as_json("anjuke_city_infor.json")

    browser = tool.open_chrome(use_user_dir=False)
    for city_name in cities:
        if city_name not in city_infor:
            city_code = cities[city_name]
            browser.get("https://" + city_code +
                        ".fang.anjuke.com/?from=navigation")
            bs = BeautifulSoup(browser.page_source,
                               'lxml')  # 将网页转化为BeautifulSoup结构
            city_label = bs.select_one(
                "#container > div.list-contents > div.list-results > div.key-sort > div.sort-condi > span > em"
            )
            if city_label is not None:
                city_num = city_label.text
            else:
                city_num = 0
                if bs.select_one(
                        "#header > div.site-search > div.sel-city > a > span"
                ) is None:
                    print("可能出现反爬虫,请处理...")
                    time.sleep(300)
                    break

            city_num = int(city_num)
            city_infor[city_name] = city_num

            tool.file.write_json("anjuke_city_infor.json", city_infor)

            time.sleep(2)
Пример #2
0
def crawler():
    browser = tool.open_chrome(use_user_dir=False)

    account_list = tool.file.load_as_string("huya_account_list.txt")
    for account_url in account_list.split("\n"):
        browser.get(account_url)

        # 读取直播间订阅数量
        text_subscribe = ""
        try:
            label_subscribe = browser.find_element_by_xpath(
                '//*[@id="activityCount"]')
            if label_subscribe is not None:
                text_subscribe = label_subscribe.text
        except NoSuchElementException:
            pass

        # 读取直播间ID
        text_id = ""
        try:
            label_id = browser.find_element_by_css_selector(
                '#J_roomHeader > div.room-hd-l > div.host-info > div.host-detail.J_roomHdDetail > span.host-rid'
            )
            if label_id is not None:
                text_id = label_id.text
        except NoSuchElementException:
            pass

        print(account_url, text_id, text_subscribe)

        time.sleep(3)
Пример #3
0
def crawler():
    browser = tool.open_chrome(use_user_dir=False)
    browser.get("https://cc.julive.com/project/s")

    bs = BeautifulSoup(browser.page_source, 'lxml')  # 将网页转化为BeautifulSoup结构

    city_dict = dict()
    for element_city in bs.select(
            "body > div.container-5-2.container > div.header-v5.header-v5-2.header-normal > div > div.inn-p > div.city-position.city-tip > div.city-change-list-new > ul > li > ul > li> a"
    ):
        city_name = element_city.text
        city_url = element_city["href"]
        city_dict[city_name] = city_url
        print(city_name, city_url)

    tool.file.write_json("julive_city_url_20191217.json", city_dict)
Пример #4
0
def crawler_city_list():
    """
    采集城市编码列表
    """
    browser = tool.open_chrome(use_user_dir=False)
    browser.get("https://www.anjuke.com/sy-city.html")

    bs = BeautifulSoup(browser.page_source, 'lxml')  # 将网页转化为BeautifulSoup结构
    city_dict = dict()
    for city_label in bs.select(
            "body > div.content > div > div.letter_city > ul > li > div > a"):
        city_name = city_label.get_text()
        city_code = city_label["href"].replace("https://",
                                               "").replace(".anjuke.com", "")
        city_dict[city_name] = city_code
        print(city_name, city_code)

    tool.file.write_json("anjuke_city_code.json", city_dict)
Пример #5
0
def crawler(live_list_path):
    browser = tool.open_chrome(use_user_dir=False)

    account_list = tool.file.load_as_string(live_list_path)
    for account_url in account_list.split("\n"):
        browser.get(account_url)

        time.sleep(3)

        text_subscribe = ""

        for _ in range(10):
            try:
                label_subscribe = browser.find_element_by_xpath(
                    '//*[@id="js-player-title"]/div/div[4]/div/span')
                if label_subscribe.text is not None and label_subscribe.text != "":
                    text_subscribe = label_subscribe.text
                    break
                time.sleep(1)
            except NoSuchElementException:
                time.sleep(1)

        print(account_url, text_subscribe)
Пример #6
0
                tweet_info["retweets"] = tweet_retweets
                tweet_info["likes"] = tweet_likes
                tweet_list.append(tweet_info)

        # 向下滚动到最下面的一条推文
        if last_label_tweet is not None:
            driver.execute_script("arguments[0].scrollIntoView();", last_label_tweet)  # 滑动到推文标签
            time.sleep(1)
        else:
            break

    return tweet_list


if __name__ == "__main__":
    selenium = tool.open_chrome()  # 打开Selenium控制的Chrome浏览器
    mySQL = tool.mysql_connect("Huabang")  # 构造MySQL数据库连接对象

    if "Huabang" in env.DATA and "Media List" in env.DATA["Huabang"]:
        for media_item in env.DATA["Huabang"]["Media List"]:
            # if media_item[0] < 440:
            #     continue
            print("开始抓取媒体:", media_item[1], "(", media_item[0], ")", "-", media_item[3], "(", media_item[2], ")")
            tweet_template = {
                "media_id": media_item[0],
                "media_name": media_item[1],
                "tweet_id": None,
                "is_retweet": 0,
                "time": None,
                "text": None,
                "replies": None,
Пример #7
0
def crawler(live_name, live_url, mysql):
    browser = tool.open_chrome(use_user_dir=False)  # 打开Chrome浏览器
    browser.get(live_url)  # 访问目标Bilibili主播的直播间
    time.sleep(10)

    time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))

    table_name = "bilibili_{}".format(time_string)
    sql_create = "CREATE TABLE live_barrage.`bilibili_{}` (" \
                 "`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
                 "`type` varchar(60) DEFAULT NULL COMMENT '弹幕类型'," \
                 "`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
                 " `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
                 " `user_id` int(11) DEFAULT NULL COMMENT '弹幕发布者等级'," \
                 " `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
                 " PRIMARY KEY (`bid`)" \
                 ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Bilibili弹幕({})';"
    mysql.create(sql_create.format(time_string, live_name))

    print("开始抓取Bilibili直播弹幕.....")

    total_time = 0
    total_num = 0

    update_time = 0

    data_id_max = 0
    for num in range(int(36000 / 0.5)):

        start_time = time.time()

        label_html = browser.find_element_by_id(
            "chat-history-list").get_attribute("outerHTML")

        soup = BeautifulSoup(label_html, 'lxml')  # 将网页内容解析为Soup对象

        barrage_list = []
        for label in soup.select("#chat-history-list > div"):

            barrage_info = {
                "type": "",  # 弹幕所属类型
                "user_name": "",  # 弹幕发布者名称
                "user_id": 0,  # 弹幕发布者等级
                "content": "",  # 弹幕内容
            }

            type_class = label["class"]

            if "danmaku-item" in type_class:
                barrage_info["type"] = "NORMAL"
            elif "welcome-msg" in type_class:
                barrage_info["type"] = "ENTER"
            elif "system-msg" in type_class:
                barrage_info["type"] = "MESSAGE"
            elif "gift-item" in type_class:
                barrage_info["type"] = "GIFT"

            if barrage_info.get("type") == "NORMAL":
                temp_time = int(label["data-ts"])
                if temp_time >= update_time:
                    update_time = temp_time
                    barrage_info["user_name"] = label["data-uname"]
                    barrage_info["user_id"] = label["data-uid"]
                    barrage_info["content"] = label["data-danmaku"]
                    barrage_list.append(barrage_info)

        mysql.insert_pure(table_name, barrage_list)

        total_num += 1
        total_time += 1000 * (time.time() - start_time)

        wait_time = 0.5
        if wait_time > (time.time() - start_time):
            time.sleep(0.5 - (time.time() - start_time))

        data_id_max += len(barrage_list)

        print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
              ")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
              round(total_time), "/", total_num, ")")
Пример #8
0
                  following_count)
            template["following_count"] = following_count
        if abs(template["followers_count"] - followers_count) > 1000 \
                or (abs(template["followers_count"] - followers_count) > 0 and template["followers_count"] < 10000):
            print("修正关注者数量:", template["followers_count"], "→",
                  followers_count)
            template["followers_count"] = followers_count
    except TypeError:
        print("修改正在关注、关注者数量失败,最终取值:正在关注 =", template["following_count"],
              "、关注数 =", template["followers_count"])

    return template


if __name__ == "__main__":
    selenium = tool.open_chrome(use_user_dir=False)  # 打开Selenium控制的Chrome浏览器
    mySQL = tool.mysql_connect("Huabang")  # 构造MySQL数据库连接对象

    if "Huabang" in env.DATA and "Media List" in env.DATA["Huabang"]:
        for media_item in env.DATA["Huabang"]["Media List"]:
            if media_item[0] < 133:
                continue
            print("开始抓取媒体:", media_item[1], "(", media_item[0], ")", "-",
                  media_item[3], "(", media_item[2], ")")
            user_template = {
                "media_id": media_item[0],
                "media_name": media_item[1],
                "name": None,
                "username": None,
                "birthday": None,
                "biography": None,
Пример #9
0
def crawler(live_name, live_url, mysql):
    browser = tool.open_chrome(use_user_dir=False)
    browser.get(live_url)  # 访问目标虎牙主播的直播间

    time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))
    table_name = "huya_{}".format(time_string)

    sql_create = "CREATE TABLE live_barrage.`huya_{}` (" \
                 "`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
                 "`type` char(10) DEFAULT NULL COMMENT '弹幕类型'," \
                 "`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
                 " `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
                 " `user_noble` int(11) DEFAULT NULL COMMENT '弹幕发布者贵族等级'," \
                 " `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
                 " `gift_name` varchar(40) DEFAULT NULL COMMENT '赠送礼物名称'," \
                 " `gift_num` int(11) DEFAULT '0' COMMENT '赠送礼物数量'," \
                 " `other` varchar(60) DEFAULT NULL COMMENT '弹幕其他信息'," \
                 " PRIMARY KEY (`bid`)" \
                 ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='虎牙弹幕({})';"
    mysql.create(sql_create.format(time_string, live_name))

    print("开始抓取虎牙直播弹幕.....")

    total_time = 0
    total_num = 0

    data_id_max = 0
    for num in range(int(36000 / 0.5)):

        start_time = time.time()

        label_html = browser.find_element_by_id(
            "chat-room__list").get_attribute("innerHTML")
        bs = BeautifulSoup(label_html, 'lxml')  # 将网页内容解析为Soup对象

        barrage_list = []
        for label in bs.select("li"):
            data_id = int(label["data-id"])  # 提取:弹幕ID

            if data_id <= data_id_max:  # 依据弹幕的ID判断弹幕是否还未抓取
                if data_id > data_id_max - 101:
                    continue
            data_id_max = data_id

            barrage_info = {
                "bid": data_id,  # 弹幕ID
                "type": "",  # 弹幕所属类型
                "user_name": "",  # 弹幕发布者名称
                "user_noble": 0,  # 弹幕发布者贵族等级
                "content": "",  # 弹幕内容
                "gift_name": "",  # 礼物名称
                "gift_num": 0,  # 礼物数量
                "other": ""  # 其他信息
            }

            category = str(label.select_one("li > div")["class"])  # 提取:弹幕类型
            if "msg-smog" in category:  # 处理smog类型弹幕(普通弹幕)
                barrage_info["type"] = "SG"
                barrage_info["user_name"] = label.select_one(
                    "li > div > span:nth-child(1)").text
                barrage_info["content"] = label.select_one(
                    "li > div > span:nth-child(3)").text
            elif "msg-normal" in category:  # 处理普通类型弹幕(普通弹幕)
                barrage_info["type"] = "NM"
                barrage_info["user_name"] = label.select_one(
                    "li > div > span:nth-child(2)").text
                barrage_info["content"] = label.select_one(
                    "li > div > span:nth-child(5)").text
            elif "msg-nobleEnter" in category:  # 处理nobleEnter类型弹幕(贵族进入弹幕)
                barrage_info["type"] = "NE"
                barrage_info["user_name"] = label.select_one(
                    "li > div > div > p > span:nth-child(1)").text
                barrage_info["user_noble"] = label.select_one(
                    "li > div > div")["class"]
                barrage_info["content"] = "驾临直播间"
            elif "msg-nobleSpeak" in category:  # 处理nobleSpeak类型弹幕(贵族发言)
                barrage_info["type"] = "NS"
                barrage_info["user_name"] = label.select_one(
                    "li > div > p > span:nth-child(2)").text
                barrage_info["user_noble"] = int(
                    label.select_one("li > div")["class"])
                barrage_info["content"] = label.select_one(
                    "li > div > p > span:nth-child(5)").text
            elif "tit-h-send" in category:  # 处理send类型提示(礼物赠送提示)
                barrage_info["type"] = "SD"
                barrage_info["user_name"] = label.select_one(
                    "li > div > span:nth-child(1)").text
                barrage_info["gift_name"] = label.select_one(
                    "li > div > span:nth-child(3) > img")["alt"]
                barrage_info["gift_num"] = int(
                    label.select_one(
                        "li > div > span:nth-child(4) > img").text)
            elif "msg-onTVLottery" in category:
                barrage_info["type"] = "TV"
                barrage_info["user_name"] = label.select_one(
                    "li > div > span:nth-child(2)").text
                barrage_info["content"] = label.select_one(
                    "li > div > div > span").text
            elif "msg-auditorSys" in category:  # 处理msg-auditorSys类型提示(系统提示)
                barrage_info["type"] = "AS"
                barrage_info["other"] = label.text
            elif "msg-sys" in category:  # 处理msg-sys类型提示(系统提示)
                barrage_info["type"] = "SY"
                barrage_info["other"] = label.text
            else:  # 处理其他类型
                barrage_info.update(type="OT", other="弹幕名称" + category)
            barrage_list.append(barrage_info)

        mysql.insert(table_name, barrage_list)

        total_num += 1
        total_time += 1000 * (time.time() - start_time)

        wait_time = 0.5
        if wait_time > (time.time() - start_time):
            time.sleep(0.5 - (time.time() - start_time))

        print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
              ")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
              round(total_time), "/", total_num, ")")
Пример #10
0
def crawler(live_name, live_url, mysql):
    browser = tool.open_chrome(use_user_dir=False)  # 打开Chrome浏览器
    browser.get(live_url)  # 访问目标斗鱼主播的直播间
    time.sleep(10)

    time_string = time.strftime("%Y%m%d_%H%M", time.localtime(time.time()))
    table_name = "douyu_{}".format(time_string)

    sql_create = "CREATE TABLE live_barrage.`douyu_{}` (" \
                 "`bid` int(11) NOT NULL AUTO_INCREMENT COMMENT '弹幕ID(barrage id)'," \
                 "`type` varchar(60) DEFAULT NULL COMMENT '弹幕类型'," \
                 "`fetch_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '弹幕抓取时间(约等于弹幕发布时间)'," \
                 " `user_name` varchar(40) DEFAULT NULL COMMENT '弹幕发布者名称'," \
                 " `user_level` int(11) DEFAULT NULL COMMENT '弹幕发布者等级'," \
                 " `content` varchar(100) DEFAULT NULL COMMENT '弹幕内容'," \
                 " `text` varchar(100) DEFAULT NULL COMMENT '弹幕其他信息'," \
                 " PRIMARY KEY (`bid`)" \
                 ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='斗鱼弹幕({})';"
    mysql.create(sql_create.format(time_string, live_name))

    print("开始抓取斗鱼直播弹幕.....")

    total_time = 0
    total_num = 0
    # screenshot = 0

    barrage_id_list = list()

    data_id_max = 0
    for num in range(int(36000 / 0.5)):

        start_time = time.time()

        label_html = browser.find_element_by_id(
            "js-barrage-list").get_attribute("innerHTML")
        soup = BeautifulSoup(label_html, 'lxml')  # 将网页内容解析为Soup对象

        barrage_list = []
        for label in soup.select("li"):

            bid = str(label["id"])  # 提取:弹幕ID

            if bid in barrage_id_list:
                continue
            barrage_id_list.append(bid)

            if len(barrage_id_list) > 200:
                barrage_id_list.remove(barrage_id_list[0])

            barrage_info = {
                "type": "",  # 弹幕所属类型
                "user_name": "",  # 弹幕发布者名称
                "user_level": 0,  # 弹幕发布者等级
                "content": "",  # 弹幕内容
                "text": ""  # 其他信息
            }

            type_class = label.select_one("li > div")["class"]
            if "Barrage-notice" in type_class and "normalBarrage" not in type_class:
                barrage_info["type"] = "NOTICE"
            elif "normalBarrage" in type_class:
                barrage_info["type"] = "NORMAL"
            elif "Barrage-userEnter" in type_class:
                barrage_info["type"] = "ENTER"
            elif "Barrage-message" in type_class:
                barrage_info["type"] = "MESSAGE"

            for info_label in label.select("li > div > span"):
                info_label_class = info_label["class"]
                if "UserLevel" in info_label_class:
                    barrage_info["user_level"] = re.search(
                        "[0-9]+", info_label["title"]).group()
                elif "Barrage-nickName" in info_label_class:
                    barrage_info["user_name"] = info_label.text.replace(
                        " ", "")
                elif "Barrage-content" in info_label_class:
                    barrage_info["content"] = info_label.text.replace(" ", "")
                elif "Barrage-text" in info_label_class:
                    barrage_info["text"] = info_label.text.replace(" ", "")

            barrage_list.append(barrage_info)

        if len(barrage_list) < 200:

            mysql.insert(table_name, barrage_list)

            total_num += 1
            total_time += 1000 * (time.time() - start_time)

            print("本次时间范围内新增弹幕:", len(barrage_list), "条,", "(共计:", data_id_max,
                  ")", "|", "运行时间:", round(total_time / total_num), "毫秒", "(",
                  round(total_time), "/", total_num, ")")

        else:

            total_num += 1
            total_time += 1000 * (time.time() - start_time)

            print("本次时间范围内弹幕列表未自动向下滚动...")

        wait_time = 0.5
        if wait_time > (time.time() - start_time):
            time.sleep(0.5 - (time.time() - start_time))

        data_id_max += len(barrage_list)