async def checkSubscriptionsUpdate(self): await self.checkCookie() cookie_map: list = vrConfig.getConfig('iwara').get('cookies').split( '=') cookie = {cookie_map[0]: cookie_map[1]} new_videos = await self.getUserVideos('', '/subscriptions', cookie) return new_videos
async def getVideoTask(self): tasks = [] if not vrConfig.getConfig('iwara').get(self.user): for user in self.following: task = asyncio.create_task(self.getUserVideos(user)) await asyncio.wait([task], timeout=1) await asyncio.wait(tasks)
async def checkCookie(self): cookie_str = vrConfig.getConfig('iwara').get('cookies') cookie_exp = vrConfig.getConfig('iwara').get('cookies_expire') if not cookie_str or time.time() > cookie_exp + 15 * 24 * 60 * 60: logger.info('cookie expired fetching') cookie = await self.fetchCookie() logger.info('new cookie fetched') vrConfig.getConfig('iwara').set('cookies', cookie) vrConfig.getConfig('iwara').set('cookies_expire', time.time()) await vrConfig.save('iwara')
async def fetchCookie(self): login_key = await self.loginKey() data = { 'name': self.user, 'pass': vrConfig.getConfig(self.ripper).get('password'), 'form_build_id': "form-jacky", 'form_id': 'user_login', 'antibot_key': login_key, 'op': '%E3%83%AD%E3%82%B0%E3%82%A4%E3%83%B3' } connector: ProxyConnector = await IwaraRipper.getConnector() async with aiohttp.ClientSession(connector=connector) as session: async with session.post("https://ecchi.iwara.tv/user/login", data=data) as _: cookie = session.cookie_jar.filter_cookies( "https://ecchi.iwara.tv/user/login") if connector: await connector.close() logger.info('fetching cookie ' + cookie.__str__()[12:]) return cookie.__str__()[12:]
async def getConnector() -> ProxyConnector: connector: ProxyConnector = ProxyConnector() proxy = vrConfig.getConfig('setting').get('proxy') if proxy: connector = ProxyConnector.from_url(proxy) return connector
def __init__(self): self.following = vrConfig.getConfig(self.ripper).get('following') self.user_videos = vrConfig.getConfig(self.ripper).get('user_videos') self.id2users: dict = vrConfig.getConfig(self.ripper).get('id2users')
class IwaraRipper: host = 'https://ecchi.iwara.tv' url = 'https://ecchi.iwara.tv/users/Alcatraz_Mentha/following' download_api = 'https://ecchi.iwara.tv/api/video/' ripper = vrConfig.getConfig('setting').get('ripper') user = vrConfig.getConfig(ripper).get('user') def __init__(self): self.following = vrConfig.getConfig(self.ripper).get('following') self.user_videos = vrConfig.getConfig(self.ripper).get('user_videos') self.id2users: dict = vrConfig.getConfig(self.ripper).get('id2users') async def getFollowing(self, user_link): # /users/Alcatraz_Mentha'必须从主页开始, my_id_info = await self.userId(user_link, False) await self.getFollowingList(my_id_info[0], my_id_info[1]) async def getFollowingList(self, my_id: str, user_follow_link: str) -> 'list': connector: ProxyConnector = await IwaraRipper.getConnector() find = ['div', 'field-content'] logger.info('checking following ' + self.host + user_follow_link) raw_data = await request('GET', self.host + user_follow_link, connector=connector) soup = BeautifulSoup(raw_data, "lxml") sear = soup.find_all(find[0], class_=find[1]) for user_element in sear: user_link: str = user_element.contents[0]['href'] user_avatar_link: str = user_element.contents[0].contents[0]['src'] if 'pictures/picture-' in user_avatar_link: user_id = '/user/' + user_avatar_link.split('-')[1] else: user_id = await self.userId(user_link) if not self.following.get(my_id): self.following[my_id] = [] self.following[my_id].append(user_id) page_find = ['li', 'pager-next'] pager = soup.find_all(page_find[0], class_=page_find[1]) if pager: next_page = pager[0].contents[0]['href'] await self.getFollowingList(my_id, next_page) else: await vrConfig.save('iwara') async def userId(self, user, only_id: bool = True): if user in self.id2users.values() and only_id: rev: dict = dict(zip(self.id2users.values(), self.id2users.keys())) return rev[user] else: uid_info = await IwaraRipper.getUserId(user, only_id) uid = uid_info if only_id else uid_info[0] logger.info(f'fetched {user} の real id {uid}') self.id2users[uid] = user await vrConfig.save('iwara') return uid_info async def checkFollowing(self): await self.getFollowing(f'/users/{self.user}/following') @staticmethod async def loginKey(): logger.info('getting login key') html = await request('GET', 'https://ecchi.iwara.tv/user/login') full = BeautifulSoup(html, "html.parser") h = full.find("head") capture = h.find("script", text=re.compile("antibot")).string.strip() start = capture.find("\"key\":") + 7 # "key":" end = capture.find("\"", start) return capture[start:end] @staticmethod async def getUserId(user_link: str, only_id: bool): logger.info(f'getting [{user_link}] real id') html = await request('GET', IwaraRipper.host + user_link) full = BeautifulSoup(html, "html.parser") h = full.find("head") capture = h.find("script", text=re.compile("views")).string.strip() start = capture.find("\"view_path\":") + 13 end = capture.find("\"", start) result = '/' + capture[start:end].replace('\\', '') if not only_id: following = full.find('div', class_='more-link').contents[1]['href'] result = [result, following] return result async def checkCookie(self): cookie_str = vrConfig.getConfig('iwara').get('cookies') cookie_exp = vrConfig.getConfig('iwara').get('cookies_expire') if not cookie_str or time.time() > cookie_exp + 15 * 24 * 60 * 60: logger.info('cookie expired fetching') cookie = await self.fetchCookie() logger.info('new cookie fetched') vrConfig.getConfig('iwara').set('cookies', cookie) vrConfig.getConfig('iwara').set('cookies_expire', time.time()) await vrConfig.save('iwara') async def fetchCookie(self): login_key = await self.loginKey() data = { 'name': self.user, 'pass': vrConfig.getConfig(self.ripper).get('password'), 'form_build_id': "form-jacky", 'form_id': 'user_login', 'antibot_key': login_key, 'op': '%E3%83%AD%E3%82%B0%E3%82%A4%E3%83%B3' } connector: ProxyConnector = await IwaraRipper.getConnector() async with aiohttp.ClientSession(connector=connector) as session: async with session.post("https://ecchi.iwara.tv/user/login", data=data) as _: cookie = session.cookie_jar.filter_cookies( "https://ecchi.iwara.tv/user/login") if connector: await connector.close() logger.info('fetching cookie ' + cookie.__str__()[12:]) return cookie.__str__()[12:] async def getVideoTask(self): tasks = [] if not vrConfig.getConfig('iwara').get(self.user): for user in self.following: task = asyncio.create_task(self.getUserVideos(user)) await asyncio.wait([task], timeout=1) await asyncio.wait(tasks) async def checkSubscriptionsUpdate(self): await self.checkCookie() cookie_map: list = vrConfig.getConfig('iwara').get('cookies').split( '=') cookie = {cookie_map[0]: cookie_map[1]} new_videos = await self.getUserVideos('', '/subscriptions', cookie) return new_videos async def getUserVideos(self, user_link, videos: str = '/videos', cookie=None) -> 'list': co: str = user_link + videos connector: ProxyConnector = await IwaraRipper.getConnector() logger.info('get user videos ' + self.host + co) raw_data = await request('GET', self.host + co, cookies=cookie, connector=connector) soup = BeautifulSoup(raw_data, "lxml") find = ['div', 'node node-video node-teaser node-teaser clearfix'] sear = soup.find_all(find[0], class_=find[1]) new_videos = [] for video_col_element in sear: is_public = 0 if video_col_element.contents[3].attrs['class'][ 0] == 'private-video' else 1 user_link = video_col_element.contents[-2]['href'] author = video_col_element.contents[-2].string user_id = await self.userId(user_link) find = ['div', 'field-item even'] video_element = video_col_element.find_all(find[0], class_=find[1]) video_link = video_element[0].contents[0]['href'] is_video = 0 if 'images' in video_link else 1 video_thumbnail = video_element[0].contents[0].contents[0]['src'] is_youtube = 1 if 'youtube' in video_thumbnail else 0 if not self.user_videos.get(user_id): self.user_videos[user_id] = {} if not self.user_videos[user_id].get(video_link): video_dict = dict(video_id=video_link[8:], author=author, public=is_public, video=is_video, url=video_link, thumbnail=video_thumbnail, youtube=is_youtube) new_videos.append([user_id, video_link]) self.user_videos[user_id][video_link] = video_dict await vrConfig.save('iwara') return new_videos async def getVideo(self, video_link) -> 'list': connector: ProxyConnector = await IwaraRipper.getConnector() raw_data = await request('GET', self.host + video_link, connector=connector) logger.info('get video info ' + self.host + video_link) soup = BeautifulSoup(raw_data, "lxml") find = ['div', 'node node-video node-full clearfix'] sear = soup.find_all(find[0], class_=find[1])[0] info_find = ['div', 'node-info'] video_info_node = sear.find_all(info_find[0], class_=info_find[1]) video_title = video_info_node[0].contents[1].contents[3].string video_time = video_info_node[0].contents[1].contents[6].strip() description_find = [ 'div', 'field field-name-body field-type-text-with-summary field-label-hidden' ] description_node = video_info_node[0].find_all( description_find[0], class_=description_find[1]) video_info = '' if description_node: video_infos = description_node[0].contents[0].contents[0].contents[ 0].contents for info in video_infos: if isinstance(info, str): video_info += info else: try: video_info += info.string except TypeError: pass user_info_node = video_info_node[0].contents[1].contents[5] user_link = user_info_node['href'] user_id = await self.userId(user_link) video = self.user_videos[user_id][video_link] video['title'] = video_title video['info'] = video_info video['time'] = video_time if not video['youtube'] or not video['video']: thumbnail_find = ['video', 'video-js vjs-default-skin hidden'] video_thumbnail = sear.find_all( thumbnail_find[0], class_=thumbnail_find[1])[0]['poster'] connector: ProxyConnector = await IwaraRipper.getConnector() links = await requestText(self.download_api + video_link[8:], connector=connector, raw=False) for i, _ in enumerate(links[0]): links[0][i]['uri'] = 'https:' + links[0][i]['uri'] video_links = links[0] video['thumbnail'] = 'https:' + video_thumbnail video['downloads'] = video_links await vrConfig.save('iwara') return video @staticmethod async def getConnector() -> ProxyConnector: connector: ProxyConnector = ProxyConnector() proxy = vrConfig.getConfig('setting').get('proxy') if proxy: connector = ProxyConnector.from_url(proxy) return connector