def download_media(info): """Get the actual media by the given info.""" # If we are supposed to use youtube-dl, try using it. # Videos with sound or youtube videos can be downloaded # more easily with youtube-dl if info.type == "video" and info.youtube_dl: log("--- Try youtube-dl") info, media = download_youtube_dl_media(info) # We got some media return it if media is not None: return info, media # If youtube-dl failed, continue and try a direct download log("--- youtube-dl failed") elif info.type == "audio" and info.youtube_dl: log("--- Try music download via youtube-dl") info, media = download_youtube_dl_music(info) # We got some media return it if media is not None: return info, media # Fail hard if this fails log("--- Music download via youtube-dl failed") return None, None log(f"--- Downloading media directly: {info.url}") request = Request(info.url, headers=headers) response = urlopen(request) media = response.read() return info, media
async def replace_vreddit_link(event): """Handle v.redd.it links.""" text = event.message.message splitted = text.split("\n") if len(splitted) == 1: url = splitted[0] elif len(splitted) == 2: url = splitted[1] elif len(splitted) > 2: return response = requests.get(url, headers=headers, allow_redirects=False) url = response.headers["Location"] response = requests.get(url, headers=headers, allow_redirects=False) url = response.headers["Location"] try: info, media = handle_reddit_web_url(url) if info is None or media is None: return await handle_file_backup(event, info, media) await handle_file_upload(event, info, media) except Exception as e: log(f"Got exception: {e}") pass
def get_media_info(payload, info): """Get the information of the media from the payload.""" data = payload[0]["data"]["children"][0]["data"] if "crosspost_parent_list" in data: data = data["crosspost_parent_list"][0] info.title = data["title"] # Reddit hosted images if data["domain"] == "i.redd.it": return info_from_ireddit(info, data["url"]) # Reddit hosted videos elif data["domain"] == "v.redd.it": return info_from_vreddit(info, data["media"]["reddit_video"]["fallback_url"]) # Gfycat videos elif data["domain"] == "gfycat.com": return info_from_gfycat(info, data["url"]) # Giphy videos elif data["domain"] == "media.giphy.com": return info_from_giphy(info, data["url"]) # Youtube video elif data["domain"] == "youtu.be": return info_from_youtube(info, data["url"]) # Imgur elif data["domain"] in ["i.imgur.com", "imgur.com"]: return info_from_imgur(info, data["url"]) log(f"--- Failed to detect media type") return None
def info_from_gfycat(info, url): """Populate info object with info from gfycat.com url.""" log("--- Detected gfycat") response = requests.get(url) soup = BeautifulSoup(response.text, features="html.parser") container = soup.find("div", {"class": "video-container"}) video = container.find("video") sources = video.children url = None for source in sources: if source["src"].startswith("https://giant.gfycat") and source["src"].endswith( ".mp4" ): url = source["src"] break if url is None: return info.url = url info.type = "video" info.extension = "mp4" return info
def info_from_youtube(info, url): """Populate info object with info from youtube.com url.""" log("--- Detected youtube") info.youtube_dl_url = url info.type = "video" info.extension = "mp4" info.youtube_dl = True return info
def info_from_vreddit(info, url): """Populate info object with info from v.redd.it url.""" log("--- Detected reddit video") info.url = url info.type = "video" info.extension = "mp4" info.youtube_dl = True return info
async def delete_media_chat(event): """Delete the current media chat id.""" try: config["bot"]["meme_chat_id"] = "" with open(config_path, "w") as file_descriptor: toml.dump(config, file_descriptor) await event.respond(f"Media chat unset") except Exception as e: log(f"Got exception: {e}")
def info_from_giphy(info, url): """Populate info object with info from *.giphy.com url.""" log("--- Detected giphy") url = url.replace("media.giphy.com", "i.giphy.com") url = url.replace("media1.giphy.com", "i.giphy.com") url = url.replace("media2.giphy.com", "i.giphy.com") url = url.replace("giphy.gif", "giphy.mp4") info.url = url info.type = "video" info.extension = "mp4" return info
async def set_media_chat(event): """Set the media chat.""" try: chat_id, peer_type = get_peer_information(event.message.to_id) log(f"Setting media chat: {chat_id}") config["bot"]["meme_chat_id"] = chat_id with open(config_path, "w") as file_descriptor: toml.dump(config, file_descriptor) await event.respond(f"Chat id set to {chat_id}") except Exception as e: log(f"Got exception: {e}")
async def replace_reddit_post_link(event): """Replace a reddit link with the actual media of the reddit link.""" try: url = event.message.message info, media = handle_reddit_web_url(url) if info is None or media is None: return await handle_file_backup(event, info, media) await handle_file_upload(event, info, media) except Exception as e: log(f"Got exception: {e}") pass
async def download_clip(event): text = event.message.message info = Info() info.type = "video" info.youtube_dl = True info.youtube_dl_url = text.split(" ")[1] try: info, media = download_media(info) await handle_file_backup(event, info, media) await handle_file_upload(event, info, media) except Exception as e: log(f"Got exception: {e}")
def info_from_ireddit(info, url): """Populate info object with info from i.redd.it url.""" log("--- Detected reddit image") info.url = url info.type = "image" if info.url.endswith(".jpg"): info.extension = "jpg" elif info.url.endswith(".png"): info.extension = "png" elif info.url.endswith(".gif"): info.extension = "gif" elif info.url.endswith(".gifv"): info.extension = "gifv" return info
async def youtube_music(event): """Set the media chat.""" text = event.message.message url = text.split(" ")[1] info = Info() info.type = "audio" info.caption = f"Original link: {url}" info.youtube_dl = True info.youtube_dl_url = url try: info, media = download_media(info) await handle_file_upload(event, info, media) except Exception as e: log(f"Got exception: {e}")
def info_from_imgur(info, url): """Populate info object with info from *.imgur.com url.""" # Gif/gifv if url.endswith(".gifv") or url.endswith(".gif"): log("--- Detected imgur gif") # We replace the .gifv and .gif, since imgur supports mp4 anyway url = url.replace("gifv", "mp4") url = url.replace("gif", "mp4") info.url = url info.type = "video" info.extension = "mp4" # Images elif url.endswith(".png"): log("--- Detected imgur png") info.url = url info.type = "image" info.extension = "png" elif url.endswith(".jpg"): log("--- Detected imgur jpg") info.url = url info.type = "image" info.extension = "jpg" return info
async def handle_file_upload(event, info, media): """Telethon file upload related logic.""" log("Handle telethon stuff:") log(f"--- Upload: {info.title}") file_handle = await bot.upload_file( media, file_name=f"{info.title}.{info.extension}") me = await bot.get_me() from_id, _ = get_sender_information(event) # Allow to have a different caption than file title if info.caption is not None: caption = info.caption else: caption = info.title # Send the file to the chat and replace the message # if the message was send by yourself if from_id == me.id: log("--- Send to original chat") await bot.send_file( event.message.to_id, file=file_handle, caption=caption, ) log("--- Delete original message") await event.message.delete() # Send the file to a meme chat if it's specified chat_id, chat_type = get_peer_information(event.message.to_id) meme_chat_id = config["bot"]["meme_chat_id"] if meme_chat_id != "" and meme_chat_id != chat_id: log("--- Send to meme chat") await bot.send_file( meme_chat_id, file=file_handle, caption=caption, )
async def download_direct_link(event, function): """Generic download class for forwarded messages.""" try: text = event.message.message info = Info() log(f"Got link: {text}") splitted = text.split("\n") if len(splitted) == 1: function(info, splitted[0]) now = datetime.now() info.title = None elif len(splitted) == 2: info.title = splitted[0] function(info, splitted[1]) elif len(splitted) > 2: return info, media = download_media(info) await handle_file_backup(event, info, media) await handle_file_upload(event, info, media) except Exception as e: log(f"Got exception: {e}") raise e
def download_youtube_dl_media(info): """Try to download a clip via youtube-dl.""" random_hash = secrets.token_hex(nbytes=8) hash = "reddit_" + random_hash options = { "outtmpl": f"/tmp/{hash}_%(title)s.%(ext)s", "quiet": True, "restrictfilenames": True, } # Try to download the media with youtube-dl log(f"--- Downloading via youtube_dl: {info.youtube_dl_url}") try: ydl = youtube_dl.YoutubeDL(options) yd_info = ydl.extract_info(info.youtube_dl_url) # Remove invalid chars that are removed from the title by youtube_dl title = sanitize_filename(yd_info["title"], True) info.extension = yd_info["ext"] path = f"/tmp/{hash}_{title}.{yd_info['ext']}" # youtube-dl might produce mkv containers, if the downloaded formats don't match # However there's no indicator that this is happening. # Check if the target file doesn't exist as mp4, but rather as mkv. # If that's the case, convert to mp4 via ffmpeg mkv_path = f"/tmp/{hash}_{title}.mkv" if not os.path.exists(path) and os.path.exists(mkv_path): path = f"/tmp/{hash}_{title}.mp4" os.system(f"ffmpeg -i '{mkv_path}' -c copy '{path}'") os.remove(mkv_path) info.extension = "mp4" # Read in RAM and remove the oiginal file with open(path, "rb") as file: media = file.read() os.remove(path) log("--- Got media") return info, media except Exception as e: log("--- Failed to use youtube-dl") print(e) return info, None
def download_youtube_dl_music(info): """Try to extract the audio of a clip via youtube-dl.""" random_hash = secrets.token_hex(nbytes=8) hash = "reddit_" + random_hash options = { "outtmpl": f"/tmp/{hash}_%(title)s.%(ext)s", "quiet": True, "format": "bestaudio", "restrictfilenames": True, } # Try to download the media with youtube-dl log(f"--- Downloading song ia youtube_dl: {info.youtube_dl_url}") try: ydl = youtube_dl.YoutubeDL(options) yd_info = ydl.extract_info(info.youtube_dl_url) # Remove invalid chars that are removed from the title by youtube_dl title = sanitize_filename(yd_info["title"], True) # Convert the webm to mp3 temp_path = f"/tmp/{hash}_{title}.{yd_info['ext']}" target_path = f"/tmp/{hash}_{title}.mp3" os.system(f"ffmpeg -i '{temp_path}' -q:a 0 -map a '{target_path}'") info.extension = "mp3" # Read in RAM and remove the oiginal files with open(target_path, "rb") as file: media = file.read() os.remove(temp_path) os.remove(target_path) log("--- Got media") return info, media except Exception as e: log("--- Failed to use youtube-dl") print(e) return info, None
async def backup_file(bot, event, info, media): """Backup the media to a file.""" # Compile file name today = date.today().isoformat() file_name = f"{today}_{info.title}.{info.extension}" log(f"--- File name: {file_name}") from_id, _ = get_sender_information(event) print(from_id) # Get username user = await bot.get_entity(from_id) username = get_username(user) # Compile path base_path = os.path.expanduser(config["bot"]["backup_path"]) dir_path = os.path.join(base_path, username) file_path = os.path.join(dir_path, file_name) log(f"--- Path : {file_path}") os.makedirs(dir_path, mode=0o755, exist_ok=True) # Write to disk with open(file_path, "wb") as media_file: media_file.write(media) log(f"--- Saved to disk!")
def handle_reddit_web_url(url): """Download media from reddit from a given web url.""" log("\nGet media info from reddit") info = Info() info.youtube_dl_url = url if not url.endswith(".json"): url += ".json" info.json_url = url # Get the json information from reddit log(f"--- Download Json: {url}") request = Request(url, headers=headers) response = urlopen(request) data = response.read().decode("utf-8") payload = json.loads(data) # Extract the media information from the payload try: info = get_media_info(payload, info) except Exception as e: log(f"--- Got exception: {e}") raise e # Check if we got some kind of info if info is None: return None, None log("--- Got media info:") log(f"--- {pprint.pformat(info)}") log("Get media:") info, media = download_media(info) return info, media
async def handle_file_backup(event, info, media): """Backup the file to the disk, if config says so.""" if config["bot"]["backup"]: log("Backing up media to disk") await backup_file(bot, event, info, media)