def test_template(self):
site_object = ChatDownloader()
try:
params = test['params']
if not params.get('logging'): # if it is not set, make it 'none'
params['logging'] = 'none'
expected_result = test.pop('expected_result', None)
if not params:
self.assertFalse('No parameters specified.') # Invalid test
messages_list = []
try:
chat = site_object.get_chat(**params)
# Ensure the site created matches the test site
if site is not BaseChatDownloader:
self.assertEqual(
chat.site.__class__.__name__, site.__name__)
messages_list = list(chat)
except Exception as e:
error = expected_result.get('error')
self.assertTrue(error is not None and isinstance(e, error))
messages_condition = expected_result.get('messages_condition')
if messages_condition:
if callable(messages_condition):
self.assertTrue(messages_condition(messages_list))
else:
self.assertFalse('Message check is not callable.')
actual_result = {
'message_types': [],
'action_types': []
}
types_to_check = [
key for key in actual_result if key in expected_result]
if types_to_check:
for message in messages_list:
message_type = message.get('message_type')
if message_type not in actual_result['message_types']:
actual_result['message_types'].append(message_type)
action_type = message.get('action_type')
if action_type not in actual_result['action_types']:
actual_result['action_types'].append(action_type)
for check in types_to_check:
self.assertCountEqual(expected_result.get(
check), actual_result.get(check))
finally:
site_object.close()
def validate(self):
video_id: str = self.entry.get()
if 'http' in video_id or 'twitch.tv' in video_id:
video_id = parse_url(video_id)
if len(video_id) > 0 and video_exists(video_id):
self.chat_downloader = ChatDownloader(video_id, overwrite_cache=self.overwrite_cache_var.get())
self.chat_downloader.start()
self.after(1, self.download)
else:
self.status_var.set('Error: Invalid URL or video ID.')
self.button.config(state=NORMAL)
def crawl(url, output):
chat = ChatDownloader().get_chat(url) # create a generator
with open(output, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(["author", "message", "time_in_seconds", "time_text", "timestamp"])
for message in chat:
writer.writerow([message["author"]["name"], message["message"], message["time_in_seconds"], message["time_text"], message["timestamp"]])
print(chat.format(message)) # print the formatted message
print("Finish!")
def _get_one_message(self, expected_error=None, **init_params):
session = ChatDownloader(**init_params)
try:
url = 'https://www.youtube.com/watch?v=5qap5aO4i9A'
chat = list(session.get_chat(url, max_messages=1))
self.assertEqual(len(chat), 1)
except Exception as e:
self.assertTrue(expected_error is not None
and isinstance(e, expected_error))
finally:
session.close()
def get_chats(id):
url = f"https://www.youtube.com/watch?v={id}"
print(url)
chat = ChatDownloader().get_chat(url)
timestamps = []
buffer = ""
for m in chat:
# print(chat.format(m))
if "!timestamp" in m["message"]:
timestamps.append(m)
# print(chat.format(m))
for m in timestamps:
buffer = buffer + chat.format(m) + "\n"
# Return the response in json format
print(buffer)
def get_twitch_chat(video_id):
url = 'https://www.twitch.tv/videos/' + str(video_id)
chat = ChatDownloader().get_chat(url)
messages = []
chat_emotes = []
emotes_images = {}
for item in chat:
messages.append({
'timestamp': item['time_in_seconds'],
'author': item['author']['name'],
'message': item['message']
})
if item.get('emotes'):
for e in item['emotes']:
chat_emotes.append({
'timestamp': item['time_in_seconds'],
'id': e['id']
})
if e['id'] not in emotes_images.keys():
emotes_images[e['id']] = e['images'][1]['url']
return messages, chat_emotes, emotes_images
def __init__(self, url, output_file, start_timestamp=None):
self.url = url
self.output_file = output_file
with open(self.output_file, "a+", encoding="utf8") as f:
data = {"time": time.time(), "url": url}
if start_timestamp:
data["startTimestamp"] = start_timestamp
data = json.dumps(data, ensure_ascii=False)
f.write(f"# {data}\n")
self.chat = ChatDownloader().get_chat(
url,
message_groups='all',
inactivity_timeout=const.CHAT_INACTIVITY_DURATION)
self.timer = utils.RepeatedTimer(const.CHAT_BUFFER_TIME,
self.__save_chat)
self.buffer = [[], []]
self.buffer_index = 0
self.is_stop = False
self.download_task = threading.Thread(target=self.__download_chat,
daemon=True)
self.download_task.start()
def retrieve_chat(url, sc_only):
# https://github.com/xenova/chat-downloader/issues/85#issuecomment-813662554
with YoutubeDL({'cookiefile': 'cookies.ytdl', 'skip_download': True}) as ytdl:
ytdl.download([url])
cookies = open('cookies.ytdl', 'r').read().replace('\t0\t', '\t{:.0f}\t'.format(time() + 60 * 60 * 24 * 90))
open('cookies.ytdl', 'w').write(cookies)
message_types = ['paid_message'] if sc_only else None
try:
chat = ChatDownloader(cookies='cookies.ytdl').get_chat(url, message_types=message_types)
# TODO: insert empty chat check
messages = []
for message in chat:
messages.append({
'time': message.get('time_in_seconds'),
'time_text': message.get('time_text'),
'amount': message.get('money', {}).get('text'),
'author': message.get('author', {}).get('name'),
'message': message.get('message')
})
return messages
except NoChatReplay as ex:
return str(ex)
class DownloadPopup(Toplevel):
def __init__(self, master, info: dict, message_store: MessageStore, video_id: str = None):
Toplevel.__init__(self, master)
self.bind('<Escape>', lambda _: self.cancel())
self.bind('<Return>', lambda _: self.ok())
self.title('Get VOD')
self.transient(master)
self.grab_set()
self.info: dict = info
self.message_store: MessageStore = message_store
self.chat_downloader: ChatDownloader = None
self.updated_info: bool = False
self.status_var = StringVar(value='...')
self.content = Frame(self)
self.content.pack(padx=20, pady=15)
self.video_title_var = StringVar(value='')
self.download_info_var = StringVar(value='')
self.eta_var = StringVar(value='')
Label(self.content, text='Enter a VOD URL or video ID:').pack(side=TOP, anchor=W, pady=(0, 5))
self.entry = Entry(self.content, width=50)
self.entry.pack(side=TOP, padx=2, pady=(0, 5))
Label(self.content, textvariable=self.status_var).pack(side=TOP, anchor=W, pady=(0, 5))
self.progress_var = IntVar(value=0)
self.progress = Progressbar(self.content, variable=self.progress_var, maximum=1)
self.progress.pack(side=TOP, fill=X, padx=2)
Label(self.content, textvariable=self.video_title_var).pack(side=TOP, anchor=W, pady=(0, 5))
Label(self.content, textvariable=self.download_info_var).pack(side=TOP, anchor=W, pady=(0, 5))
Label(self.content, textvariable=self.eta_var).pack(side=TOP, anchor=W, pady=(0, 5))
self.overwrite_cache_var = BooleanVar(value=False)
self.overwrite_cache_check = Checkbutton(self.content, text='Overwrite cache',
variable=self.overwrite_cache_var)
self.overwrite_cache_check.pack(side=TOP, anchor=W, pady=(0, 5))
self.button = Button(self.content, text='OK', command=self.ok)
self.button.pack(side=TOP)
self.update()
x_coord = self.master.winfo_x() + (self.master.winfo_width() // 2) - (self.winfo_width() // 2)
y_coord = self.master.winfo_y() + (self.master.winfo_height() // 2) - (self.winfo_height() // 2)
self.geometry(f'{self.winfo_width()}x{self.winfo_height()}+{x_coord}+{y_coord}')
self.entry.focus_set()
self.protocol('WM_DELETE_WINDOW', self.cancel)
if video_id:
self.entry.insert(0, video_id)
self.overwrite_cache_check.focus_set()
chat_filename: str = os.path.join(CACHE_FOLDER, f'chat-{video_id}.json')
if not os.path.exists(chat_filename):
self.ok()
def cancel(self):
if self.chat_downloader:
self.chat_downloader.kill()
self.info.clear()
self.info.update({'title': 'Chat Player'})
self.destroy()
def ok(self):
self.button.config(state=DISABLED)
self.overwrite_cache_check.config(state=DISABLED)
self.status_var.set('Validating...')
self.after(1, self.validate)
def validate(self):
video_id: str = self.entry.get()
if 'http' in video_id or 'twitch.tv' in video_id:
video_id = parse_url(video_id)
if len(video_id) > 0 and video_exists(video_id):
self.chat_downloader = ChatDownloader(video_id, overwrite_cache=self.overwrite_cache_var.get())
self.chat_downloader.start()
self.after(1, self.download)
else:
self.status_var.set('Error: Invalid URL or video ID.')
self.button.config(state=NORMAL)
def download(self):
if not self.chat_downloader.info:
self.status_var.set('Getting info')
self.after(100, self.download)
elif not self.chat_downloader.messages:
if not self.updated_info:
self.status_var.set('Downloading chat')
self.info.update(self.chat_downloader.info)
self.video_title_var.set(self.info.get('title'))
self.updated_info = True
self.progress_var.set(self.chat_downloader.progress)
self.download_info_var.set(
f'{self.chat_downloader.num_messages} messages downloaded. '
f'Duration {self.chat_downloader.duration_done_str}/{self.chat_downloader.duration_str}.')
self.eta_var.set(f'ETA: {self.chat_downloader.eta_str}')
self.after(100, self.download)
else:
self.message_store.set_messages(self.chat_downloader.messages)
self.destroy()