def test_template(self): site_object = ChatDownloader() try: params = test['params'] if not params.get('logging'): # if it is not set, make it 'none' params['logging'] = 'none' expected_result = test.pop('expected_result', None) if not params: self.assertFalse('No parameters specified.') # Invalid test messages_list = [] try: chat = site_object.get_chat(**params) # Ensure the site created matches the test site if site is not BaseChatDownloader: self.assertEqual( chat.site.__class__.__name__, site.__name__) messages_list = list(chat) except Exception as e: error = expected_result.get('error') self.assertTrue(error is not None and isinstance(e, error)) messages_condition = expected_result.get('messages_condition') if messages_condition: if callable(messages_condition): self.assertTrue(messages_condition(messages_list)) else: self.assertFalse('Message check is not callable.') actual_result = { 'message_types': [], 'action_types': [] } types_to_check = [ key for key in actual_result if key in expected_result] if types_to_check: for message in messages_list: message_type = message.get('message_type') if message_type not in actual_result['message_types']: actual_result['message_types'].append(message_type) action_type = message.get('action_type') if action_type not in actual_result['action_types']: actual_result['action_types'].append(action_type) for check in types_to_check: self.assertCountEqual(expected_result.get( check), actual_result.get(check)) finally: site_object.close()
def validate(self): video_id: str = self.entry.get() if 'http' in video_id or 'twitch.tv' in video_id: video_id = parse_url(video_id) if len(video_id) > 0 and video_exists(video_id): self.chat_downloader = ChatDownloader(video_id, overwrite_cache=self.overwrite_cache_var.get()) self.chat_downloader.start() self.after(1, self.download) else: self.status_var.set('Error: Invalid URL or video ID.') self.button.config(state=NORMAL)
def crawl(url, output): chat = ChatDownloader().get_chat(url) # create a generator with open(output, 'w', newline='') as csvfile: writer = csv.writer(csvfile) writer.writerow(["author", "message", "time_in_seconds", "time_text", "timestamp"]) for message in chat: writer.writerow([message["author"]["name"], message["message"], message["time_in_seconds"], message["time_text"], message["timestamp"]]) print(chat.format(message)) # print the formatted message print("Finish!")
def _get_one_message(self, expected_error=None, **init_params): session = ChatDownloader(**init_params) try: url = 'https://www.youtube.com/watch?v=5qap5aO4i9A' chat = list(session.get_chat(url, max_messages=1)) self.assertEqual(len(chat), 1) except Exception as e: self.assertTrue(expected_error is not None and isinstance(e, expected_error)) finally: session.close()
def get_chats(id): url = f"https://www.youtube.com/watch?v={id}" print(url) chat = ChatDownloader().get_chat(url) timestamps = [] buffer = "" for m in chat: # print(chat.format(m)) if "!timestamp" in m["message"]: timestamps.append(m) # print(chat.format(m)) for m in timestamps: buffer = buffer + chat.format(m) + "\n" # Return the response in json format print(buffer)
def get_twitch_chat(video_id): url = 'https://www.twitch.tv/videos/' + str(video_id) chat = ChatDownloader().get_chat(url) messages = [] chat_emotes = [] emotes_images = {} for item in chat: messages.append({ 'timestamp': item['time_in_seconds'], 'author': item['author']['name'], 'message': item['message'] }) if item.get('emotes'): for e in item['emotes']: chat_emotes.append({ 'timestamp': item['time_in_seconds'], 'id': e['id'] }) if e['id'] not in emotes_images.keys(): emotes_images[e['id']] = e['images'][1]['url'] return messages, chat_emotes, emotes_images
def __init__(self, url, output_file, start_timestamp=None): self.url = url self.output_file = output_file with open(self.output_file, "a+", encoding="utf8") as f: data = {"time": time.time(), "url": url} if start_timestamp: data["startTimestamp"] = start_timestamp data = json.dumps(data, ensure_ascii=False) f.write(f"# {data}\n") self.chat = ChatDownloader().get_chat( url, message_groups='all', inactivity_timeout=const.CHAT_INACTIVITY_DURATION) self.timer = utils.RepeatedTimer(const.CHAT_BUFFER_TIME, self.__save_chat) self.buffer = [[], []] self.buffer_index = 0 self.is_stop = False self.download_task = threading.Thread(target=self.__download_chat, daemon=True) self.download_task.start()
def retrieve_chat(url, sc_only): # https://github.com/xenova/chat-downloader/issues/85#issuecomment-813662554 with YoutubeDL({'cookiefile': 'cookies.ytdl', 'skip_download': True}) as ytdl: ytdl.download([url]) cookies = open('cookies.ytdl', 'r').read().replace('\t0\t', '\t{:.0f}\t'.format(time() + 60 * 60 * 24 * 90)) open('cookies.ytdl', 'w').write(cookies) message_types = ['paid_message'] if sc_only else None try: chat = ChatDownloader(cookies='cookies.ytdl').get_chat(url, message_types=message_types) # TODO: insert empty chat check messages = [] for message in chat: messages.append({ 'time': message.get('time_in_seconds'), 'time_text': message.get('time_text'), 'amount': message.get('money', {}).get('text'), 'author': message.get('author', {}).get('name'), 'message': message.get('message') }) return messages except NoChatReplay as ex: return str(ex)
class DownloadPopup(Toplevel): def __init__(self, master, info: dict, message_store: MessageStore, video_id: str = None): Toplevel.__init__(self, master) self.bind('<Escape>', lambda _: self.cancel()) self.bind('<Return>', lambda _: self.ok()) self.title('Get VOD') self.transient(master) self.grab_set() self.info: dict = info self.message_store: MessageStore = message_store self.chat_downloader: ChatDownloader = None self.updated_info: bool = False self.status_var = StringVar(value='...') self.content = Frame(self) self.content.pack(padx=20, pady=15) self.video_title_var = StringVar(value='') self.download_info_var = StringVar(value='') self.eta_var = StringVar(value='') Label(self.content, text='Enter a VOD URL or video ID:').pack(side=TOP, anchor=W, pady=(0, 5)) self.entry = Entry(self.content, width=50) self.entry.pack(side=TOP, padx=2, pady=(0, 5)) Label(self.content, textvariable=self.status_var).pack(side=TOP, anchor=W, pady=(0, 5)) self.progress_var = IntVar(value=0) self.progress = Progressbar(self.content, variable=self.progress_var, maximum=1) self.progress.pack(side=TOP, fill=X, padx=2) Label(self.content, textvariable=self.video_title_var).pack(side=TOP, anchor=W, pady=(0, 5)) Label(self.content, textvariable=self.download_info_var).pack(side=TOP, anchor=W, pady=(0, 5)) Label(self.content, textvariable=self.eta_var).pack(side=TOP, anchor=W, pady=(0, 5)) self.overwrite_cache_var = BooleanVar(value=False) self.overwrite_cache_check = Checkbutton(self.content, text='Overwrite cache', variable=self.overwrite_cache_var) self.overwrite_cache_check.pack(side=TOP, anchor=W, pady=(0, 5)) self.button = Button(self.content, text='OK', command=self.ok) self.button.pack(side=TOP) self.update() x_coord = self.master.winfo_x() + (self.master.winfo_width() // 2) - (self.winfo_width() // 2) y_coord = self.master.winfo_y() + (self.master.winfo_height() // 2) - (self.winfo_height() // 2) self.geometry(f'{self.winfo_width()}x{self.winfo_height()}+{x_coord}+{y_coord}') self.entry.focus_set() self.protocol('WM_DELETE_WINDOW', self.cancel) if video_id: self.entry.insert(0, video_id) self.overwrite_cache_check.focus_set() chat_filename: str = os.path.join(CACHE_FOLDER, f'chat-{video_id}.json') if not os.path.exists(chat_filename): self.ok() def cancel(self): if self.chat_downloader: self.chat_downloader.kill() self.info.clear() self.info.update({'title': 'Chat Player'}) self.destroy() def ok(self): self.button.config(state=DISABLED) self.overwrite_cache_check.config(state=DISABLED) self.status_var.set('Validating...') self.after(1, self.validate) def validate(self): video_id: str = self.entry.get() if 'http' in video_id or 'twitch.tv' in video_id: video_id = parse_url(video_id) if len(video_id) > 0 and video_exists(video_id): self.chat_downloader = ChatDownloader(video_id, overwrite_cache=self.overwrite_cache_var.get()) self.chat_downloader.start() self.after(1, self.download) else: self.status_var.set('Error: Invalid URL or video ID.') self.button.config(state=NORMAL) def download(self): if not self.chat_downloader.info: self.status_var.set('Getting info') self.after(100, self.download) elif not self.chat_downloader.messages: if not self.updated_info: self.status_var.set('Downloading chat') self.info.update(self.chat_downloader.info) self.video_title_var.set(self.info.get('title')) self.updated_info = True self.progress_var.set(self.chat_downloader.progress) self.download_info_var.set( f'{self.chat_downloader.num_messages} messages downloaded. ' f'Duration {self.chat_downloader.duration_done_str}/{self.chat_downloader.duration_str}.') self.eta_var.set(f'ETA: {self.chat_downloader.eta_str}') self.after(100, self.download) else: self.message_store.set_messages(self.chat_downloader.messages) self.destroy()