Пример #1
0
    async def feed(self, amount: int, lang: str = "en"):
        page = await self.client.new_page(
            blocked_resources=["media", "image", "font"])

        logger.debug('📨 Request "Trending" page')

        result: List[dict] = []

        pbar = tqdm(total=amount, desc=f"📈 Getting trending {lang.upper()}")

        page.on(
            "response",
            lambda res: asyncio.create_task(
                catch_response_and_store(res, result)),
        )
        _ = await self.client.goto(
            "/foryou",
            params={"lang": lang},
            page=page,
            options={"waitUntil": "networkidle0"},
        )
        logger.debug('📭 Got response from "Trending" page')

        while len(result) < amount:

            logger.debug("🖱 Trying to scroll to last video item")
            await page.evaluate(
                """
                document.querySelector('.video-feed-container > .lazyload-wrapper:last-child')
                    .scrollIntoView();
            """, )
            await page.waitFor(1_000)

            elements = await page.JJ(".video-feed-item")
            logger.debug(f"🔎 Found {len(elements)} items for clear")

            pbar.n = min(len(result), amount)
            pbar.refresh()

            if len(elements) < 500:
                logger.debug("🔻 Too less for clearing page")
                continue

            await page.JJeval(
                ".video-feed-container > .lazyload-wrapper:not(:last-child)",
                pageFunction=
                "(elements) => elements.forEach(el => el.remove())",
            )
            logger.debug(f"🎉 Cleaned {len(elements) - 1} items from page")
            await page.waitFor(30_000)

        await page.close()
        pbar.close()
        return result[:amount]
Пример #2
0
    async def feed(self, username: str, amount: int):
        page: Page = await self.client.new_page(blocked_resources=["image", "media", "font"])
        logger.debug(f"ЁЯУи Request {username} feed")

        result: List[dict] = []
        user_info_queue: asyncio.Queue = asyncio.Queue(maxsize=1)

        page.on(
            "response", lambda res: asyncio.create_task(catch_response_and_store(res, result)),
        )

        page.on(
            "response",
            lambda res: asyncio.create_task(
                catch_response_info(res, user_info_queue, "/user/detail"),
            ),
        )
        _ = await self.client.goto(f"/{username}", page=page, options={"waitUntil": "networkidle0"})
        logger.debug(f"ЁЯУн Got {username} feed")

        await page.waitForSelector(".video-feed-item", options={"visible": True})

        user_info = await user_info_queue.get()
        user_video_count = user_info["userInfo"]["stats"]["videoCount"]

        if user_video_count < amount:
            logger.info(
                f"тЪая╕П  User {username} has only {user_video_count} videos. "
                f"Set amount from {amount} to {user_video_count}",
            )
            amount = user_video_count

        pbar = tqdm(total=amount, desc=f"ЁЯУИ Getting {username} feed")
        pbar.n = min(len(result), amount)
        pbar.refresh()

        attempts = 0
        last_result = len(result)

        while len(result) < amount:
            logger.debug("ЁЯЦ▒ Trying to scroll to last video item")
            await page.evaluate(
                """
                document.querySelector('.video-feed-item:last-child')
                    .scrollIntoView();
            """,
            )
            await page.waitFor(1_000)

            elements = await page.JJ(".video-feed-item")
            logger.debug(f"ЁЯФО Found {len(elements)} items for clear")

            pbar.n = min(len(result), amount)
            pbar.refresh()

            if last_result == len(result):
                attempts += 1
            else:
                attempts = 0

            if attempts > 10:
                pbar.clear()
                pbar.total = len(result)
                logger.info(
                    f"тЪая╕П  After 10 attempts found {len(result)} videos. "
                    f"Probably some videos are private",
                )
                break

            last_result = len(result)

            if len(elements) < 500:
                logger.debug("ЁЯФ╗ Too less for clearing page")
                continue

            await page.JJeval(
                ".video-feed-item:not(:last-child)",
                pageFunction="(elements) => elements.forEach(el => el.remove())",
            )
            logger.debug(f"ЁЯОЙ Cleaned {len(elements) - 1} items from page")
            await page.waitFor(30_000)

        await page.close()
        pbar.close()
        return result[:amount]
Пример #3
0
    async def feed(self, username: str, amount: int):
        page: Page = await self.client.new_page(blocked_resources=["image", "media", "font"])
        logger.debug(f"📨 Request {username} feed")

        result: List[dict] = []

        page.on(
            "response",
            lambda res: asyncio.create_task(catch_response_and_store(res, result)),
        )

        _ = await self.client.goto(f"/{username}", page=page, options={"waitUntil": "networkidle0"})
        logger.debug(f"📭 Got {username} feed")

        await page.waitForSelector(".video-feed-item", options={"visible": True})

        pbar = tqdm(total=amount, desc=f"📈 Getting {username} feed")
        pbar.n = min(len(result), amount)
        pbar.refresh()

        attempts = 0
        last_result = len(result)

        while len(result) < amount:
            logger.debug("🖱 Trying to scroll to last video item")
            await page.evaluate(
                """
                document.querySelector('.video-feed-item:last-child')
                    .scrollIntoView();
            """,
            )
            await page.waitFor(1_000)

            elements = await page.JJ(".video-feed-item")
            logger.debug(f"🔎 Found {len(elements)} items for clear")

            pbar.n = min(len(result), amount)
            pbar.refresh()

            if last_result == len(result):
                attempts += 1
            else:
                attempts = 0

            if attempts > 10:
                pbar.clear()
                pbar.total = len(result)
                logger.info(
                    f"⚠️  After 10 attempts found {len(result)} videos. "
                    f"Probably some videos are private",
                )
                break

            last_result = len(result)

            if len(elements) < 500:
                logger.debug("🔻 Too less for clearing page")
                continue

            await page.JJeval(
                ".video-feed-item:not(:last-child)",
                pageFunction="(elements) => elements.forEach(el => el.remove())",
            )
            logger.debug(f"🎉 Cleaned {len(elements) - 1} items from page")
            await page.waitFor(30_000)

        await page.close()
        pbar.close()
        return result[:amount]