def download_media(info):
"""Get the actual media by the given info."""
# If we are supposed to use youtube-dl, try using it.
# Videos with sound or youtube videos can be downloaded
# more easily with youtube-dl
if info.type == "video" and info.youtube_dl:
log("--- Try youtube-dl")
info, media = download_youtube_dl_media(info)
# We got some media return it
if media is not None:
return info, media
# If youtube-dl failed, continue and try a direct download
log("--- youtube-dl failed")
elif info.type == "audio" and info.youtube_dl:
log("--- Try music download via youtube-dl")
info, media = download_youtube_dl_music(info)
# We got some media return it
if media is not None:
return info, media
# Fail hard if this fails
log("--- Music download via youtube-dl failed")
return None, None
log(f"--- Downloading media directly: {info.url}")
request = Request(info.url, headers=headers)
response = urlopen(request)
media = response.read()
return info, media
async def replace_vreddit_link(event):
"""Handle v.redd.it links."""
text = event.message.message
splitted = text.split("\n")
if len(splitted) == 1:
url = splitted[0]
elif len(splitted) == 2:
url = splitted[1]
elif len(splitted) > 2:
return
response = requests.get(url, headers=headers, allow_redirects=False)
url = response.headers["Location"]
response = requests.get(url, headers=headers, allow_redirects=False)
url = response.headers["Location"]
try:
info, media = handle_reddit_web_url(url)
if info is None or media is None:
return
await handle_file_backup(event, info, media)
await handle_file_upload(event, info, media)
except Exception as e:
log(f"Got exception: {e}")
pass
def get_media_info(payload, info):
"""Get the information of the media from the payload."""
data = payload[0]["data"]["children"][0]["data"]
if "crosspost_parent_list" in data:
data = data["crosspost_parent_list"][0]
info.title = data["title"]
# Reddit hosted images
if data["domain"] == "i.redd.it":
return info_from_ireddit(info, data["url"])
# Reddit hosted videos
elif data["domain"] == "v.redd.it":
return info_from_vreddit(info,
data["media"]["reddit_video"]["fallback_url"])
# Gfycat videos
elif data["domain"] == "gfycat.com":
return info_from_gfycat(info, data["url"])
# Giphy videos
elif data["domain"] == "media.giphy.com":
return info_from_giphy(info, data["url"])
# Youtube video
elif data["domain"] == "youtu.be":
return info_from_youtube(info, data["url"])
# Imgur
elif data["domain"] in ["i.imgur.com", "imgur.com"]:
return info_from_imgur(info, data["url"])
log(f"--- Failed to detect media type")
return None
def info_from_gfycat(info, url):
"""Populate info object with info from gfycat.com url."""
log("--- Detected gfycat")
response = requests.get(url)
soup = BeautifulSoup(response.text, features="html.parser")
container = soup.find("div", {"class": "video-container"})
video = container.find("video")
sources = video.children
url = None
for source in sources:
if source["src"].startswith("https://giant.gfycat") and source["src"].endswith(
".mp4"
):
url = source["src"]
break
if url is None:
return
info.url = url
info.type = "video"
info.extension = "mp4"
return info
def info_from_youtube(info, url):
"""Populate info object with info from youtube.com url."""
log("--- Detected youtube")
info.youtube_dl_url = url
info.type = "video"
info.extension = "mp4"
info.youtube_dl = True
return info
def info_from_vreddit(info, url):
"""Populate info object with info from v.redd.it url."""
log("--- Detected reddit video")
info.url = url
info.type = "video"
info.extension = "mp4"
info.youtube_dl = True
return info
async def delete_media_chat(event):
"""Delete the current media chat id."""
try:
config["bot"]["meme_chat_id"] = ""
with open(config_path, "w") as file_descriptor:
toml.dump(config, file_descriptor)
await event.respond(f"Media chat unset")
except Exception as e:
log(f"Got exception: {e}")
def info_from_giphy(info, url):
"""Populate info object with info from *.giphy.com url."""
log("--- Detected giphy")
url = url.replace("media.giphy.com", "i.giphy.com")
url = url.replace("media1.giphy.com", "i.giphy.com")
url = url.replace("media2.giphy.com", "i.giphy.com")
url = url.replace("giphy.gif", "giphy.mp4")
info.url = url
info.type = "video"
info.extension = "mp4"
return info
async def set_media_chat(event):
"""Set the media chat."""
try:
chat_id, peer_type = get_peer_information(event.message.to_id)
log(f"Setting media chat: {chat_id}")
config["bot"]["meme_chat_id"] = chat_id
with open(config_path, "w") as file_descriptor:
toml.dump(config, file_descriptor)
await event.respond(f"Chat id set to {chat_id}")
except Exception as e:
log(f"Got exception: {e}")
async def replace_reddit_post_link(event):
"""Replace a reddit link with the actual media of the reddit link."""
try:
url = event.message.message
info, media = handle_reddit_web_url(url)
if info is None or media is None:
return
await handle_file_backup(event, info, media)
await handle_file_upload(event, info, media)
except Exception as e:
log(f"Got exception: {e}")
pass
async def download_clip(event):
text = event.message.message
info = Info()
info.type = "video"
info.youtube_dl = True
info.youtube_dl_url = text.split(" ")[1]
try:
info, media = download_media(info)
await handle_file_backup(event, info, media)
await handle_file_upload(event, info, media)
except Exception as e:
log(f"Got exception: {e}")
def info_from_ireddit(info, url):
"""Populate info object with info from i.redd.it url."""
log("--- Detected reddit image")
info.url = url
info.type = "image"
if info.url.endswith(".jpg"):
info.extension = "jpg"
elif info.url.endswith(".png"):
info.extension = "png"
elif info.url.endswith(".gif"):
info.extension = "gif"
elif info.url.endswith(".gifv"):
info.extension = "gifv"
return info
async def youtube_music(event):
"""Set the media chat."""
text = event.message.message
url = text.split(" ")[1]
info = Info()
info.type = "audio"
info.caption = f"Original link: {url}"
info.youtube_dl = True
info.youtube_dl_url = url
try:
info, media = download_media(info)
await handle_file_upload(event, info, media)
except Exception as e:
log(f"Got exception: {e}")
def info_from_imgur(info, url):
"""Populate info object with info from *.imgur.com url."""
# Gif/gifv
if url.endswith(".gifv") or url.endswith(".gif"):
log("--- Detected imgur gif")
# We replace the .gifv and .gif, since imgur supports mp4 anyway
url = url.replace("gifv", "mp4")
url = url.replace("gif", "mp4")
info.url = url
info.type = "video"
info.extension = "mp4"
# Images
elif url.endswith(".png"):
log("--- Detected imgur png")
info.url = url
info.type = "image"
info.extension = "png"
elif url.endswith(".jpg"):
log("--- Detected imgur jpg")
info.url = url
info.type = "image"
info.extension = "jpg"
return info
async def handle_file_upload(event, info, media):
"""Telethon file upload related logic."""
log("Handle telethon stuff:")
log(f"--- Upload: {info.title}")
file_handle = await bot.upload_file(
media, file_name=f"{info.title}.{info.extension}")
me = await bot.get_me()
from_id, _ = get_sender_information(event)
# Allow to have a different caption than file title
if info.caption is not None:
caption = info.caption
else:
caption = info.title
# Send the file to the chat and replace the message
# if the message was send by yourself
if from_id == me.id:
log("--- Send to original chat")
await bot.send_file(
event.message.to_id,
file=file_handle,
caption=caption,
)
log("--- Delete original message")
await event.message.delete()
# Send the file to a meme chat if it's specified
chat_id, chat_type = get_peer_information(event.message.to_id)
meme_chat_id = config["bot"]["meme_chat_id"]
if meme_chat_id != "" and meme_chat_id != chat_id:
log("--- Send to meme chat")
await bot.send_file(
meme_chat_id,
file=file_handle,
caption=caption,
)
async def download_direct_link(event, function):
"""Generic download class for forwarded messages."""
try:
text = event.message.message
info = Info()
log(f"Got link: {text}")
splitted = text.split("\n")
if len(splitted) == 1:
function(info, splitted[0])
now = datetime.now()
info.title = None
elif len(splitted) == 2:
info.title = splitted[0]
function(info, splitted[1])
elif len(splitted) > 2:
return
info, media = download_media(info)
await handle_file_backup(event, info, media)
await handle_file_upload(event, info, media)
except Exception as e:
log(f"Got exception: {e}")
raise e
def download_youtube_dl_media(info):
"""Try to download a clip via youtube-dl."""
random_hash = secrets.token_hex(nbytes=8)
hash = "reddit_" + random_hash
options = {
"outtmpl": f"/tmp/{hash}_%(title)s.%(ext)s",
"quiet": True,
"restrictfilenames": True,
}
# Try to download the media with youtube-dl
log(f"--- Downloading via youtube_dl: {info.youtube_dl_url}")
try:
ydl = youtube_dl.YoutubeDL(options)
yd_info = ydl.extract_info(info.youtube_dl_url)
# Remove invalid chars that are removed from the title by youtube_dl
title = sanitize_filename(yd_info["title"], True)
info.extension = yd_info["ext"]
path = f"/tmp/{hash}_{title}.{yd_info['ext']}"
# youtube-dl might produce mkv containers, if the downloaded formats don't match
# However there's no indicator that this is happening.
# Check if the target file doesn't exist as mp4, but rather as mkv.
# If that's the case, convert to mp4 via ffmpeg
mkv_path = f"/tmp/{hash}_{title}.mkv"
if not os.path.exists(path) and os.path.exists(mkv_path):
path = f"/tmp/{hash}_{title}.mp4"
os.system(f"ffmpeg -i '{mkv_path}' -c copy '{path}'")
os.remove(mkv_path)
info.extension = "mp4"
# Read in RAM and remove the oiginal file
with open(path, "rb") as file:
media = file.read()
os.remove(path)
log("--- Got media")
return info, media
except Exception as e:
log("--- Failed to use youtube-dl")
print(e)
return info, None
def download_youtube_dl_music(info):
"""Try to extract the audio of a clip via youtube-dl."""
random_hash = secrets.token_hex(nbytes=8)
hash = "reddit_" + random_hash
options = {
"outtmpl": f"/tmp/{hash}_%(title)s.%(ext)s",
"quiet": True,
"format": "bestaudio",
"restrictfilenames": True,
}
# Try to download the media with youtube-dl
log(f"--- Downloading song ia youtube_dl: {info.youtube_dl_url}")
try:
ydl = youtube_dl.YoutubeDL(options)
yd_info = ydl.extract_info(info.youtube_dl_url)
# Remove invalid chars that are removed from the title by youtube_dl
title = sanitize_filename(yd_info["title"], True)
# Convert the webm to mp3
temp_path = f"/tmp/{hash}_{title}.{yd_info['ext']}"
target_path = f"/tmp/{hash}_{title}.mp3"
os.system(f"ffmpeg -i '{temp_path}' -q:a 0 -map a '{target_path}'")
info.extension = "mp3"
# Read in RAM and remove the oiginal files
with open(target_path, "rb") as file:
media = file.read()
os.remove(temp_path)
os.remove(target_path)
log("--- Got media")
return info, media
except Exception as e:
log("--- Failed to use youtube-dl")
print(e)
return info, None
async def backup_file(bot, event, info, media):
"""Backup the media to a file."""
# Compile file name
today = date.today().isoformat()
file_name = f"{today}_{info.title}.{info.extension}"
log(f"--- File name: {file_name}")
from_id, _ = get_sender_information(event)
print(from_id)
# Get username
user = await bot.get_entity(from_id)
username = get_username(user)
# Compile path
base_path = os.path.expanduser(config["bot"]["backup_path"])
dir_path = os.path.join(base_path, username)
file_path = os.path.join(dir_path, file_name)
log(f"--- Path : {file_path}")
os.makedirs(dir_path, mode=0o755, exist_ok=True)
# Write to disk
with open(file_path, "wb") as media_file:
media_file.write(media)
log(f"--- Saved to disk!")
def handle_reddit_web_url(url):
"""Download media from reddit from a given web url."""
log("\nGet media info from reddit")
info = Info()
info.youtube_dl_url = url
if not url.endswith(".json"):
url += ".json"
info.json_url = url
# Get the json information from reddit
log(f"--- Download Json: {url}")
request = Request(url, headers=headers)
response = urlopen(request)
data = response.read().decode("utf-8")
payload = json.loads(data)
# Extract the media information from the payload
try:
info = get_media_info(payload, info)
except Exception as e:
log(f"--- Got exception: {e}")
raise e
# Check if we got some kind of info
if info is None:
return None, None
log("--- Got media info:")
log(f"--- {pprint.pformat(info)}")
log("Get media:")
info, media = download_media(info)
return info, media
async def handle_file_backup(event, info, media):
"""Backup the file to the disk, if config says so."""
if config["bot"]["backup"]:
log("Backing up media to disk")
await backup_file(bot, event, info, media)
