def YouPlay(vidUrl,isPlaylist=False):
VIDEO_URL = vidUrl
ydl_opt={"extract_flat":True,'quiet':True}
ydl = YoutubeDL(ydl_opt)
ydl.add_default_info_extractors()
if(not isPlaylist):
info = ydl.extract_info(VIDEO_URL,download=False)
title=info['title']
print(title)
url = info['formats'][0]['url']
instance = vlc.Instance('--no-video --quiet')
Vars.player = instance.media_player_new()
media = instance.media_new(url)
Vars.player.set_media(media)
Vars.player.play()
else:
ydl_opt={"extract_flat":True,'quiet':True,'playlistrandom':True}
ydl = YoutubeDL(ydl_opt)
info = ydl.extract_info(VIDEO_URL,download=False)
entries=info['entries']
Vars.playlistPlayer=PlaylistPlaying(Vars.player, entries)
Vars.playlistPlayer.start()
def _get_stream_info(self, video_id, pl_id):
info = None
log.debug("Starting YoutubeDL process")
try:
ytdl = YoutubeDL(self.ytdl_options)
url = "https://www.youtube.com/watch?v=%s" % video_id
info = ytdl.extract_info(url,
ie_key="Youtube",
download=False,
process=True)
ua = info["formats"][-1]["http_headers"]["User-Agent"]
self.session.http.headers.update({"User-Agent": ua})
pl_path = self.get_option("playlist-dir")
if pl_id and pl_path:
if not os.path.isdir(pl_path):
os.makedirs(pl_path)
log.debug(
"Playlist directory '{0}' created".format(pl_path))
url = "https://www.youtube.com/playlist?list=%s" % pl_id
_info = ytdl.extract_info(url,
ie_key="YoutubePlaylist",
download=False,
process=True)
self._save2M3U(pl_path, _info, ua)
except Exception as e:
raise PluginError(e)
return info
def process_item(self, item, spider):
fb_video_url = item.get('fb_video_url')
if fb_video_url:
ydl = YoutubeDL()
ydl.add_default_info_extractors()
ydl.extract_info(fb_video_url)
return item
def youtubeScreen2(data=None, page=0):
addHistory(youtubeScreen2, data, True)
try:
settings.ignoreInputKey=True
settings.selectorWindow.setStatusTempText("Youtube: downloading list...", 1000)
ydl = YoutubeDL({'quiet':False,'extract_flat':True,'dump_single_json':True,'logger':youtubeLogger()})
res = ydl.extract_info(data['url'])
if res['_type']== 'url':
res = ydl.extract_info(res['url'])
if res['entries'][0]['ie_key']=='YoutubePlaylist':
tracks=res['entries']
for t in tracks:
t['display'] = t['title']+" (Playlist)"
t['search'] = functions.get_initials(t['title'])
pager = pagerContent(tracks, 0, getCommon2_h_options(), youtubeScreen2)
else:
#remove "[Deleted video]" and "[Private video]"
tracks=list(filter(lambda t:not re.match('\[.* video\]',t['title']),res['entries']))
for t in tracks:
t['display'] = t['title']
t['search'] = functions.get_initials(t['title'])
t['network'] = 'youtube'
pager = pagerContent(tracks, 0, getCommon2_h_options(), playlist.addVideo)
pager.startDisplay(page, 0, 1)
except:
settings.logger.printException()
else:
settings.ignoreInputKey=False
class Downloader:
def __init__(self):
"""
Class whose instance can be used to extract audio from a URL
"""
# instance of YoutubeDownload class that can download a video using a url
# with only the url
self.video_downloader = YoutubeDL({"format": "mp4"})
# instance of YoutubeDownload class that can download audio using a url
# audio_downloader = YoutubeDL({"format": "bestaudio"}) # downloads best audio file
self.audio_downloader = YoutubeDL({"format": "m4a"})
def downloadVideo(self, url: str) -> None:
"""
Download video using a url
:param url: str
:return: None
"""
# download the video to the current directory
self.video_downloader.extract_info(url)
def downloadAudio(self, url: str) -> None:
"""
Download audio file using a url
:param url: str
:return: None
"""
# download the audio file in the current directory
self.audio_downloader.extract_info(url)
def downloadSong(self):
# Getting input from user and converting it to a format as "you+searched+this+"
EntryText = self.downloadedSong.get()
searchKeyword_temp = (EntryText).split()
searchKeyword = ""
for i in searchKeyword_temp:
searchKeyword += i + "+"
# using urllib.request.open to get the html data of the youtube search page
http = urllib.request.urlopen(
("https://www.youtube.com/results?search_query=" + searchKeyword))
# re.findall is used to find all the specific codes of the videos
# http.read().decode() gives us a string with the raw html code of the site
videoId = re.findall(r"watch\?v=(\S{11})", http.read().decode())
videoURL = "https://www.youtube.com/watch?v=" + videoId[0]
# YoutubeDl requires you to set a fromat code which is what file type you want to save it as
audioDownloader = YoutubeDL({
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}]
})
# Downloads the song in the folder containing the .py file. I have to change this will do
audioDownloader.extract_info(videoURL)
os.chdir(r"D:\Users\Vivaan Wadhwa\Documents\GitHub\CS-Project\Music")
# Fetching Songs
songtracks = os.listdir()
# Inserting Songs into Playlist
for track in songtracks:
self.playlist.insert(END, track)
def get_youtube_audio(url):
try:
audio_dl = YoutubeDL({'format':'bestaudio'})
audio_dl.extract_info(url)
except Exception:
return "오디오 처리에 실패하였습니다."
return None
def save_audio(artist, title, fileName, file_directory):
if not len(firebase_admin._apps):
cred = credentials.Certificate(
r"C:\Users\gwan1\PycharmProjects\News_Project\serviceAccountKey.json"
)
firebase_admin.initialize_app(cred)
firestore.client()
bucket = storage.bucket()
blob = bucket.blob(fileName)
tube_artist = artist.split(' ')
tube_title = title.split(' ')
play_link = 'https://www.youtube.com/results?search_query='
for title_word in tube_title:
play_link = play_link + title_word + '+'
for artist_word in tube_artist:
play_link = play_link + artist_word + '+'
play_link = play_link[:-1]
options = webdriver.ChromeOptions()
browser = webdriver.Chrome(executable_path="C:\chromedriver.exe",
options=options)
browser.get(play_link)
WebDriverWait(browser, 10).until(
expected_conditions.visibility_of_element_located((
By.XPATH,
'//*[@id="yDmH0d"]/c-wiz/div/div/div/div[2]/div[1]/div[4]/form/div[1]/div/button/span'
))).click()
urltxt = browser.page_source
soupTube = bs.BeautifulSoup(urltxt, 'html.parser')
hrefs = soupTube.find_all(
'a', {'class': 'yt-simple-endpoint style-scope ytd-video-renderer'
})[0]['href']
To_play_url = 'https://www.youtube.com' + hrefs
os.chdir("D:/TempAudFiles")
audio_downloder = YoutubeDL({'format': 'bestaudio/best'})
audio_downloder.extract_info(To_play_url)
downloaded_temp = os.listdir("D:\TempAudFiles")
for file in downloaded_temp:
if file.lower().startswith(
artist[:6].lower()) or file.lower().startswith(title.lower()):
os.rename(file, fileName.lower())
file_directory = "D:\AudFiles" + "\\" + fileName
dowloaded = os.listdir("D:\AudFiles")
if fileName not in dowloaded:
shutil.move(('D:/TempAudFiles/' + fileName.lower()),
"D:\AudFiles")
downloaded_temp = os.listdir("D:\TempAudFiles")
for file in downloaded_temp:
os.remove(file)
try:
with open(file_directory, 'rb') as audioFiles:
blob.upload_from_file(audioFiles, content_type='audio/mpeg')
except Exception as e:
print('Audio {} was not saved to storage: '.format(fileName) + ': ' +
e)
def getVideoURL(settings):
from youtube_dl import YoutubeDL
ydl_opts = {
'format': settings["format"],
'forceurl': True,
'ffmpeg_location': ffmpegPath,
'merge_output_format': 'mkv',
'outtmpl': f'{settings["downloadVideoPath"]}.%(ext)s',
"cachedir": False
}
ydl = YoutubeDL(ydl_opts)
if settings["downloadVideo"]:
ydl_info = ydl.extract_info(settings["videoURL"], download=True)
settings["downloadVideoPath"] = f'{settings["downloadVideoPath"]}.mkv'
else:
ydl_info = ydl.extract_info(settings["videoURL"], download=False)
if 'requested_formats' in ydl_info:
rf = ydl_info["requested_formats"]
videoInfo = rf[0]
else:
videoInfo = ydl_info
dashFormatIDs = []
dashVideoFormatID = None
dashAudioFormatID = None
if settings["downloadVideo"]:
settings["inputVideo"] = settings["downloadVideoPath"]
else:
if videoInfo["protocol"] == 'http_dash_segments':
settings["isDashVideo"] = True
dashVideoFormatID = videoInfo["format_id"]
dashFormatIDs.append(dashVideoFormatID)
else:
settings["videoURL"] = videoInfo["url"]
if 'requested_formats' in ydl_info:
audioInfo = rf[1]
settings["audiobr"] = int(audioInfo["tbr"])
if audioInfo["protocol"] == 'http_dash_segments':
settings["isDashAudio"] = True
dashAudioFormatID = audioInfo["format_id"]
dashFormatIDs.append(dashAudioFormatID)
else:
settings["audioURL"] = audioInfo["url"]
if dashFormatIDs:
filteredDashPath = filterDash(videoInfo["url"], dashFormatIDs)
if settings["isDashVideo"]:
settings["videoURL"] = filteredDashPath
if settings["isDashAudio"]:
settings["audioURL"] = filteredDashPath
return getVideoInfo(settings, videoInfo)
async def handler(event):
sender = await event.get_sender()
chat = await event.get_chat()
link = event.text.split(' ')[1]
links = event.text.split(' ')[2]
if os.path.exists(sender.username):
#os.makedirs(sender.username)
ydl = YoutubeDL({
'outtmpl': sender.username + "/" + links,
'add-header': "kaios/2.0"
})
info = ydl.extract_info(link, download=True)
print(info['title'])
e = os.listdir("/app/" + sender.username + "/")
filepath = "/app/" + sender.username + "/"
#async for filepath in e:
c = ""
await client.send_message(chat, info['title'])
await client.send_file(chat,
"/app/" + sender.username + "/" + c.join(e),
force_document=True)
path = os.path.join("/app/" + sender.username + "/", c.join(e))
os.remove(path)
if not os.path.exists(sender.username):
os.makedirs(sender.username)
ydl = YoutubeDL({
'outtmpl': sender.username + "/" + links,
'add-header': "kaios/2.0"
})
info = ydl.extract_info(link, download=True)
print(info['title'])
e = os.listdir("/app/" + sender.username + "/")
filepath = "/app/" + sender.username + "/"
#async for filepath in e:
c = ""
await client.send_message(chat, info['title'])
#await client.send_file(chat,"/app/"+sender.username+"/"+c.join(e),force_document=True)
path = os.path.join("/app/" + sender.username + "/", c.join(e))
if len(e) > 0:
try:
await client.send_file(chat,
"/app/" + sender.username + "/" +
c.join(e),
force_document=True)
except:
pass
await client.send_message(
chat, "sm " + "/app/" + sender.username + "/" + c.join(e))
os.remove(path)
def YTDownloader(video_url):
if not video_url.startswith("https://www.youtube.com/watch?v="):
raise ValueError(f"url is not youtube video valid url: {video_url}")
print(f"Video URL: [ {red_bold_underlined(video_url)} ]")
video_id = video_url.split("watch?v=")[1]
print(f"Video ID: {red_bold(video_id)}")
print_yellow_bold("\ndownloading...\n")
# with YoutubeDL({}) as downloader:
# downloader.download([video_url])
downloaded_video_path = downloaded_video_folder / f"{video_id}_video.%(ext)s"
downloader = YoutubeDL({"outtmpl": downloaded_video_path.absolute().as_posix()})
with downloader:
json_data = downloader.extract_info(
video_url,
download=True # can be False
)
# write_json_to_file(json_data, json_data_path)
print("\nVideo downloaded saved here:")
print_green_bold_underlined(downloaded_video_path)
def get_video_url(params):
"""Get video URL and start video player"""
url_selected = ''
all_datas_videos_quality = []
all_datas_videos_path = []
videos_html = utils.get_webcontent(params.video_url)
videos_soup = bs(videos_html, 'html.parser')
list_videos = videos_soup.find(
'ul', class_='nav nav-tabs').find_all('a')
for video in list_videos:
if '#video-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_jwplayer_id = video.get('data-jwplayer-id')
# Case mp4
if value_jwplayer_id != '':
list_streams = videos_soup.find_all(
'div', class_='jwplayer')
for stream in list_streams:
if stream.get('id') == value_jwplayer_id:
url = stream.get('data-source')
# Cas Yt
else:
video_id = re.compile(
'youtube.com/embed/(.*?)\?').findall(videos_html)[0]
url = resolver.get_stream_youtube(video_id, False)
all_datas_videos_path.append(url)
# Get link from FranceTV
elif '#ftv-player-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_ftvlayer_id = video.get('data-ftvplayer-id')
list_streams = videos_soup.find_all(
'iframe', class_='embed-responsive-item')
for stream in list_streams:
if stream.get('id') == value_ftvlayer_id:
url_id = stream.get('src')
ydl = YoutubeDL()
ydl.add_default_info_extractors()
with ydl:
result = ydl.extract_info(
url_id, download=False)
for format_video in result['formats']:
url = format_video['url']
all_datas_videos_path.append(url)
if len(all_datas_videos_quality) > 1:
seleted_item = common.sp.xbmcgui.Dialog().select(
common.GETTEXT('Choose video quality'),
all_datas_videos_quality)
if seleted_item == -1:
return ''
url_selected = all_datas_videos_path[seleted_item]
return url_selected
else:
return all_datas_videos_path[0]
class YoutubeDLWrapper(object):
""" Used to wrap youtubedl import, since youtubedl currently overrides
global HTMLParser.locatestarttagend regex with a different regex
that doesn't quite work.
This wrapper ensures that this regex is only set for YoutubeDL and unset
otherwise
"""
def __init__(self):
import HTMLParser as htmlparser
self.htmlparser = htmlparser
self.orig_tagregex = htmlparser.locatestarttagend
from youtube_dl import YoutubeDL as YoutubeDL
self.ydl_tagregex = htmlparser.locatestarttagend
htmlparser.locatestarttagend = self.orig_tagregex
self.ydl = YoutubeDL(
dict(simulate=True, youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
info = None
try:
self.htmlparser.locatestarttagend = self.ydl_tagregex
info = self.ydl.extract_info(url)
finally:
self.htmlparser.locatestarttagend = self.orig_tagregex
return info
def download_audio(input_url, filesize_limit):
input_url = input_url.split('&list', 1)[0]
ydl_opts = {
'format': 'bestaudio[protocol^=http]',
'logger': logger,
'outtmpl': DOWNLOAD_DIR + '%(title)s - %(extractor)s-%(id)s.%(ext)s',
'writethumbnail': True
}
ydl = YoutubeDL(ydl_opts)
info = ydl.extract_info(input_url, download=False)
webpage_info = info['extractor'] + " " + info['id']
info_format = next((item for item in info['formats']
if item['format_id'] == info['format_id']), None)
if _youtube_video_not_music(info):
raise WrongCategoryError(webpage_info + ": not under Music category")
audio_filesize = _get_filesize(info_format)
if int(audio_filesize) > filesize_limit:
raise WrongFileSizeError(webpage_info + ": audio file size (" +
str(audio_filesize) + ") is greater" +
"than the limit: (" + str(filesize_limit) +
")")
logger.info("%s: downloading...", webpage_info)
ydl.process_info(info)
info = _add_downloads_to_info_dict(info, ydl_opts)
if not _files_successfully_downloaded(info['downloads']):
_delete_downloaded_files(info['downloads'])
raise ValueError(webpage_info +
": failed to download audio or thumbnail")
return info
def record_stream(channel, output_dir):
channels = get_channels("world", 2)
channel_url = "http://www.youtube.com/watch?v={}".format(channels[channel -
1])
print("Downloading from", channel_url)
ydl = YoutubeDL()
ydl.add_default_info_extractors()
info = ydl.extract_info(channel_url, download=False)
if info["uploader_id"] != "coachella":
print("Uploader is not coachella, stopping")
exit()
filename = "channel{}-{}.ts".format(channel, int(time.time()))
streamlink = Streamlink()
streamlink.set_loglevel("debug")
streamlink.set_logoutput(output_dir + "/" + filename + ".log")
streamlink.set_option("hls-live-edge", 9999999)
streamlink.set_option("hls-segment-attempts", 99)
streamlink.set_option("hls-segment-threads", 5)
streamlink.set_option("hls-segment-timeout", 9999)
streams = streamlink.streams(channel_url)
stream = streams["best"]
print("Found stream {}".format(stream.url))
print("Writing stream to {}".format(output_dir + "/" + filename))
with open(output_dir + "/" + filename,
'wb') as out_file, stream.open() as in_stream:
while True:
data = in_stream.read(1024)
if not data:
return
out_file.write(data)
def admin():
if "loggedin" in session:
if request.method == "GET":
videos = Video.objects()
return render_template('admin.html', videos=videos)
elif request.method == "POST":
form = request.form
link = form['link']
ydl = YoutubeDL()
data = ydl.extract_info(link, download=False)
title = data["title"]
thumbnail = data["thumbnail"]
views = data["view_count"]
youtube_id = data["id"]
new_video = Video(title=title,
thumbnail=thumbnail,
views=views,
link=link,
youtube_id=youtube_id)
new_video.save()
return redirect(url_for("admin"))
else:
return redirect(url_for("login"))
def admin():
if "loggedin" in session:
if session['loggedin'] == True:
if request.method == 'GET':
videos = Video.objects()
return render_template('admin.html', videos=videos)
elif request.method == 'POST':
form = request.form
link = form['link']
ydl = YoutubeDL()
data = ydl.extract_info(link, download=False)
title = data['title']
views = data['view_count']
thumbnail = data['thumbnail']
youtube_id = data['id']
video = Video(title=title,
views=views,
thumbnail=thumbnail,
youtube_id=youtube_id,
link=link)
video.save()
return redirect(url_for('admin'))
else:
return "Đăng nhập chưa mà đòi"
else:
return "Đăng nhập chưa mà đòi"
def process_message(body, message):
data = None
try:
data = json.loads(body)
except json.JSONDecodeError as err:
print("Error parsing JSON payload: {0}".format(err))
ydl = YoutubeDL()
info = ydl.extract_info(data.get('url'))
# FIXME: the YoutubeDL is a great commandline tool but it lacks any kind
# of logical OO implementation hence we cannot easily manipulate any of it's
# characteristics so we can't guess what the filename is
# we are dealing with. Usually we should find a match on
# '%(title)s-%(id)s.*' so we will just filter it out of the list after
# listing the files in the directory
# filename = DEFAULT_OUTTMPL % info
match = '%(title)s-%(id)s.' % info
filename = [name for name in os.listdir() if match in name]
if len(filename) != 1:
# produce some kind of logging event about failure,
# maybe differentiate between different lenghts or content
print("shit hit the fan because we can't find the file")
message.ack()
filename = filename[0]
shutil.move(filename, '{}/{}'.format(config['DOWNLOAD_LOCATION'],
filename))
message.ack()
def extract_info(self):
opts = {
'simulate': True
}
opts.update(YTDL_PLAYER_OPTS)
ydl = YoutubeDL(opts)
return ydl.extract_info(self.resource)
def download(self, custom=None, no_part=True):
global audio_downloader
audio_downloader = YoutubeDL({
#'buffersize': 512,
#'http_chunk_size': 256,
'audioformat': 'wav',
'format': 'bestaudio',
'outtmpl': self.title + self.ext,
'extractaudio': True,
'retries': 5,
'continuedl': True,
'nopart': no_part,
'hls_prefer_native': True,
'quiet': True
})
audio_downloader.extract_info(self.url)
def download_playlist(url):
"""
download the whole playlist of the given url
NOTE: youtubedl was used instead of pytube because it gives stable result
and that's because pytube sometimes didn't return the correct name of the given link
"""
ydl = YoutubeDL({
# this line is VERY IMPORTANT because if there's private viedeos
# or anything unavilable to download youtube-dl won't crash
# and will continue to the next videos in the playlist
"ignoreerrors": True,
# to make fetching info faster because we are not interested in DASH info
"youtube_include_dash_manifest": False,
# don't print log messages in stdout
"quiet": True
})
with ydl:
playlist = ydl.extract_info(
url,
download=False # We just want to extract the info
)
name_playlist = safe_filename(playlist.get('title'))
mkdir_and_chdir(name_playlist)
download_multiple_videos(playlist, name_playlist)
async def play(self, ctx, *, search: str = None):
global vc
vc = ctx.voice_client
if not vc:
await ctx.invoke(self.connect)
vc = ctx.voice_client
await ctx.send("Getting everything ready...", delete_after=5)
ytdl = YoutubeDL(ytdl_opts)
player = ytdl.extract_info(search, download=False)
global video
if 'entries' in player:
video = player['entries'][0]
else:
video = player
vid.append(video)
await songs.put(vid)
title = video["alt_title"]
author = video["uploader"]
await ctx.send(f"Enqueued {title} from {author} successfully.")
def zap(session, service, **kwargs):
errormsg = None
if service and "http" in service.toString():
url = service.toString()
url = url.split(":")
if len(url) > 9:
url = url[10]
if YoutubeDL is not None and url.startswith("YT-DL%3a//"):
url = url.replace("YT-DL%3a//", "")
url = url.replace("%3a", ":")
try:
ydl = YoutubeDL({'format': 'best'})
result = ydl.extract_info(url, download=False)
if result and hasattr(result, "url"):
url = result['url']
print(
"[ChannelSelection] zap / YoutubeDL result url %s"
% url)
return (url, errormsg)
else:
errormsg = "No Link found!"
print("[ChannelSelection] zap / YoutubeDL no streams")
except Exception as e:
errormsg = str(e)
print("[ChannelSelection] zap / YoutubeDL failed %s" %
str(e))
pass
return (None, errormsg)
class DownloadProcess(multiprocessing.Process):
"""
Actual download process which calls youtube-dl. You should never need to interact with this directly.
"""
def __init__(self, url, status_queue, output_writer):
super(DownloadProcess, self).__init__()
self.url = url
self.status_queue = status_queue
self.info = {}
sys.stdout = output_writer
sys.stderr = output_writer
self.downloader = YoutubeDL({"progress_hooks": [self.update_status]})
self.downloader.add_default_info_extractors()
def update_status(self, status):
self.status_queue.put(status)
def run(self):
self.info = self.downloader.extract_info(self.url)
def stop(self):
# Kill any child processes that may have spawned (e.g. ffmpeg)
import psutil, signal
s = psutil.Process()
for child in s.children(recursive=True):
child.send_signal(signal.SIGINT)
def admin():
if "loggedin" in session:
if request.method == "GET":
videos = Video.objects()
return render_template('admin.html', videos=videos)
elif request.method == "POST":
form = request.form
link = form['link']
ydl = YoutubeDL()
data = ydl.extract_info(link, download=False)
title = data['title']
thumbnail = data['thumbnail']
views = data['view_count']
youtubeid = data['id']
new_video = Video(title=title,
thumbnail=thumbnail,
views=views,
youtubeid=youtubeid,
link=link)
new_video.save()
return redirect(url_for('admin'))
else:
return "Yêu cầu đăng nhập !!!"
def admin():
if request.method == "GET":
if "logged_in" in session:
videos = Video.objects()
return render_template("admin.html", videos=videos)
else:
return render_template("error.html")
else:
form = request.form
link = form["link"]
ydl = YoutubeDL()
data = ydl.extract_info(link, download=False)
title = data["title"]
thumbnail = data["thumbnail"]
views = data["view_count"]
link = data["webpage_url"]
youtubeid = data["id"]
new_video = Video(title=title,
thumbnail=thumbnail,
views=views,
link=link,
youtubeid=youtubeid)
new_video.save()
return render_template("admin.html")
class YoutubeDLDownloader(DownloaderBase):
scheme = "ytdl"
def __init__(self, extractor, output):
DownloaderBase.__init__(self, extractor, output)
options = {
"format": self.config("format") or None,
"ratelimit": text.parse_bytes(self.config("rate"), None),
"retries": self.config("retries", extractor._retries),
"socket_timeout": self.config("timeout", extractor._timeout),
"nocheckcertificate": not self.config("verify", extractor._verify),
"nopart": not self.part,
}
options.update(self.config("raw-options") or {})
if self.config("logging", True):
options["logger"] = self.log
self.ytdl = YoutubeDL(options)
def download(self, url, pathfmt):
try:
info_dict = self.ytdl.extract_info(url[5:], download=False)
except Exception:
return False
if "entries" in info_dict:
index = pathfmt.keywords.get("_ytdl_index")
if index is None:
return self._download_playlist(pathfmt, info_dict)
else:
info_dict = info_dict["entries"][index]
return self._download_video(pathfmt, info_dict)
def _download_video(self, pathfmt, info_dict):
pathfmt.set_extension(info_dict["ext"])
if pathfmt.exists():
pathfmt.temppath = ""
return True
if self.part and self.partdir:
pathfmt.temppath = os.path.join(
self.partdir, pathfmt.filename)
self.ytdl.params["outtmpl"] = pathfmt.temppath.replace("%", "%%")
self.out.start(pathfmt.path)
try:
self.ytdl.process_info(info_dict)
except Exception:
self.log.debug("Traceback", exc_info=True)
return False
return True
def _download_playlist(self, pathfmt, info_dict):
pathfmt.set_extension("%(playlist_index)s.%(ext)s")
self.ytdl.params["outtmpl"] = pathfmt.realpath
for entry in info_dict["entries"]:
self.ytdl.process_info(entry)
return True
def admin():
if "loggedin" in session:
if request.method == "GET":
videos = Video.objects()
return render_template("admin.html", videos=videos)
elif request.method == "POST":
form = request.form
link = form['Link']
ydl = YoutubeDL()
# Ham extract info
data = ydl.extract_info(link, download=False)
title = data['title']
thumbnail = data['thumbnail']
views = data['view_count']
youtube_id = data['id']
video = Video(title=title,
thumbnail=thumbnail,
view=views,
youtube_id=youtube_id,
link=link)
video.save()
return redirect(url_for("admin"))
else:
return redirect(url_for("login"))
def video():
if request.method == 'GET':
videos = Video.objects()
cardios = Cardio.objects()
# overweights = Overweight.objects()
underweights = Underweight.objects()
yogas= Yoga.objects()
exercises = Exercise.objects()
return render_template('admin.html', videos=videos, cardios=cardios, underweights=underweights, yogas=yogas, exercises=exercises )
elif request.method == 'POST':
form = request.form
link = form['link']
ydl = YoutubeDL()
data = ydl.extract_info(link, download=False)
title = data['title']
thumbnail = data['thumbnail']
youtube_id = data['id']
# duration = data['duration']
new_ex = Exercise(
title= title,
link= link,
thumbnail= thumbnail,
youtube_id= youtube_id
# duration= duration
)
new_ex.save()
return redirect(url_for('video'))
def search_db():
ydl = YoutubeDL({'ignoreerrors': True})
with open(new_videos, 'w+') as output:
for query in queries:
for page in range(1,2):
r = ydl.extract_info("ytsearchdate30:{}".format(query) , download=False)
video_list = r['entries']
for video in video_list:
if video is None: continue
p = recomendadiotn_utils.compute_prediction(video)
video_id = video.get('webpage_url', '')
data_front = {'title':video['title'], 'video_id':video_id }
data_front['score'] = float(p)
data_front['update_time'] = time.time()
output.write(f'{json.dumps(data_front)}\n')
return True
def getVideoUrl(self):
VIDEO_FMT_PRIORITY_MAP = {
1 : '38', #MP4 Original (HD)
2 : '37', #MP4 1080p (HD)
3 : '22', #MP4 720p (HD)
4 : '18', #MP4 360p
5 : '35', #FLV 480p
6 : '34', #FLV 360p
}
KEY_FORMAT_ID = u"format_id"
KEY_URL = u"url"
KEY_ENTRIES = u"entries"
KEY_FORMATS = u"formats"
video_url = None
video_id = str(self.getTubeId())
# Getting video webpage
#URLs for YouTube video pages will change from the format http://www.youtube.com/watch?v=ylLzyHk54Z0 to http://www.youtube.com/watch#!v=ylLzyHk54Z0.
watch_url = 'http://www.youtube.com/watch?v=%s' % video_id
format_prio = "/".join(VIDEO_FMT_PRIORITY_MAP.itervalues())
ytdl = YoutubeDL(params={"youtube_include_dash_manifest": False, "format" : format_prio})
result = ytdl.extract_info(watch_url, download=False)
if KEY_ENTRIES in result: # Can be a playlist or a list of videos
entry = result[KEY_ENTRIES][0] #TODO handle properly
else:# Just a video
entry = result
video_url = entry.get(KEY_URL)
return str(video_url)
def lambda_handler(event, context):
lambda_response = {
"isBase64Encoded": False,
"statusCode": 200,
"headers": {
"Content-Type": "application/json"
},
"body": "{}"
}
if not 'url' in event['queryStringParameters'] or not event[
'queryStringParameters']['url']:
lambda_response['statusCode'] = 400
return lambda_response
url = urllib.parse.unquote(event['queryStringParameters']['url'])
ydl_opts = {'skip_download': True, 'quiet': True, 'no_warnings': True}
ydl = YoutubeDL(ydl_opts)
try:
download_url = ydl.extract_info(url)
lambda_response['body'] = json.dumps(download_url)
except DownloadError:
lambda_response['statusCode'] = 404
except:
lambda_response['statusCode'] = 500
return lambda_response
def _extract(
query: str,
ie_key: str = None,
address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address] = None,
video: bool = False,
) -> dict:
option = copy.copy(YTDLOption)
if video:
option["format"] = "(best)[protocol!=http_dash_segments]"
if address:
option["source_address"] = str(address)
YoutubeDL = YoutubeDLClient(option)
Data = YoutubeDL.extract_info(query, download=False, ie_key=ie_key)
if not Data:
raise NoSearchResults
if "entries" in Data:
if len(Data["entries"]) == 1:
return Data["entries"][0]
return Data["entries"]
if not Data:
raise NoSearchResults
return Data
class YoutubeDLWrapper(object):
""" Used to wrap youtubedl import, since youtubedl currently overrides
global HTMLParser.locatestarttagend regex with a different regex
that doesn't quite work.
This wrapper ensures that this regex is only set for YoutubeDL and unset
otherwise
"""
def __init__(self):
import HTMLParser as htmlparser
self.htmlparser = htmlparser
self.orig_tagregex = htmlparser.locatestarttagend
from youtube_dl import YoutubeDL as YoutubeDL
self.ydl_tagregex = htmlparser.locatestarttagend
htmlparser.locatestarttagend = self.orig_tagregex
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
info = None
try:
self.htmlparser.locatestarttagend = self.ydl_tagregex
info = self.ydl.extract_info(url)
finally:
self.htmlparser.locatestarttagend = self.orig_tagregex
return info
def download_channel(url):
"""
download the whole channel of the given url (the url must ends with "/videos"
so you need to select the videos tab in the channel page and then copy the url
NOTE: youtubedl was used instead of pytube because it gives stable result
and that's because pytube sometimes didn't return the correct name of the given link
"""
ydl = YoutubeDL({
# this line is VERY IMPORTANT because if there's private viedeos
# or anything unavilable to download youtube-dl won't crash
# and will continue to the next videos in the playlist
"ignoreerrors": True,
# Download single video instead of a playlist if in doubt.
"noplaylist": True,
# to make fetching info faster because we are not interested in DASH info
"youtube_include_dash_manifest": False,
# don't print log messages in stdout
"quiet": True
})
with ydl:
channel = ydl.extract_info(
url,
download=False # We just want to extract the info
)
# original output without split is "Uploads from CHANNEL_NAME" (quotes not included)
# so we need to split and pick the last item
name_channel = safe_filename(channel.get("title").split(' from ')[-1])
# print(name_channel)
# print("="*50)
mkdir_and_chdir(name_channel)
download_multiple_videos(channel, name_channel)
def get_stream_selector(self):
"""Return format selector for the media URL."""
ydl = YoutubeDL({"quiet": True, "logger": _LOGGER})
try:
all_media = ydl.extract_info(self.get_media_url(), process=False)
except DownloadError:
# This exception will be logged by youtube-dl itself
raise MEDownloadException()
if "entries" in all_media:
_LOGGER.warning("Playlists are not supported, looking for the first video")
entries = list(all_media["entries"])
if entries:
selected_media = entries[0]
else:
_LOGGER.error("Playlist is empty")
raise MEDownloadException()
else:
selected_media = all_media
def stream_selector(query):
"""Find stream URL that matches query."""
try:
ydl.params["format"] = query
requested_stream = ydl.process_ie_result(selected_media, download=False)
except (ExtractorError, DownloadError):
_LOGGER.error("Could not extract stream for the query: %s", query)
raise MEQueryException()
return requested_stream["url"]
return stream_selector
class InfoParser():
def __init__(self, url=None, ydl_opt={}):
self.__url = url
self.__ydl = YoutubeDL(ydl_opt)
self.__ydl_obj = YDLObject()
@property
def url(self):
return self.__url
@url.setter
def url(self, url):
self.__url = url
@property
def ydl_object(self):
return self.__ydl_obj
def generate_info(self):
try:
info_dict = self.__ydl.extract_info(self.__url, download=False)
self.__ydl_obj.title = info_dict['title']
self.__ydl_obj.url = self.__url
for format in info_dict['formats']:
filesize = format.get('filesize') if format.get('filesize') else format.get('filesize_approx')
self.__ydl_obj.format_info.append({
'format_id': format['format'],
'extension': format['ext'],
'resolution': self.__ydl.format_resolution(format),
'filesize': format_bytes(filesize)
})
return True
except Exception as e:
print(e)
return False
class PyJizzParser(object):
def __init__(self, model):
self.model = model
self.model.parser = self
self.ydl = YoutubeDL()
self.ydl.add_default_info_extractors()
def parseCategories(self):
c = PornHubCategoryParser(self.model)
c.run()
def parseCategoryPage(self, category, page = 1):
if page == 0 or page == 1:
url = self.model.categories_url[category]
else:
url = "{site}{page_url}{page}".format(
site = self.model.site,
page_url = self.model.p**n[category]['page_url'],
page = page)
print("page parser creating for page", page)
p = PornHubPageParser(self.model, url, category, page)
p.run()
print("page parser exit for page", page)
def getInfo(self, vkey):
info = self.ydl.extract_info('http://www.pornhub.com/view_video.php?viewkey={v}'.format(v = vkey), download=False)
return info
def test_unicode_path_redirection(self):
# XXX: Python 3 http server does not allow non-ASCII header values
if sys.version_info[0] == 3:
return
ydl = YoutubeDL({'logger': FakeLogger()})
r = ydl.extract_info('http://127.0.0.1:%d/302' % self.port)
self.assertEqual(r['entries'][0]['url'], 'http://127.0.0.1:%d/vid.mp4' % self.port)
def get_youtube_info(url):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
try:
info = ydl.extract_info(url, download=False)
except:
return None
return info
def get_info(url):
ydl = YoutubeDL({
'forceurl': True,
'quiet': True
})
ydl.add_default_info_extractors()
return ydl.extract_info(url, download=False)
def v_info_url(url):
url = urllib.parse.unquote(url) # workaround
format = request.forms.get('format', 'best')
ytdl = YoutubeDL({'format': format, 'source_address': '0.0.0.0'})
info = ytdl.extract_info(url, download=False)
response.set_header('Access-Control-Allow-Origin', '*')
return info
def get_video_data(video_id):
data={}
video_url = 'https://www.youtube.com/watch?v='+video_id
ydl_opts = {
'format': 'bestaudio/mp3',
}
ydl = YoutubeDL(ydl_opts)
r = ydl.extract_info(video_url, download=False)
data['aud_url']=r['url']
ydl = YoutubeDL()
r = ydl.extract_info(video_url, download=False)
data['vid_url']=r['url']
data['title']=r['title']
data['thumbnail']=r['thumbnails'][0]['url']
return data
def test_nocheckcertificate(self):
if sys.version_info >= (2, 7, 9): # No certificate checking anyways
ydl = YoutubeDL({'logger': FakeLogger()})
self.assertRaises(
Exception,
ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
def viceLink(self, callback, url):
video_url = None
m = re.search('share_url=(.+)("|&)*', url)
if m:
url = m.group(1)
#print 'url:',url
try:
ytdl = YoutubeDL()
result = ytdl.extract_info(url, download=False)
#print 'result:',result
video_url = str(result["url"])
except Exception, e:
printl(str(e),self,"E")
def get_stream_selector(self):
"""Return format selector for the media URL."""
from youtube_dl import YoutubeDL
from youtube_dl.utils import DownloadError, ExtractorError
ydl = YoutubeDL({'quiet': True, 'logger': _LOGGER})
try:
all_media = ydl.extract_info(self.get_media_url(),
process=False)
except DownloadError:
# This exception will be logged by youtube-dl itself
raise MEDownloadException()
if 'entries' in all_media:
_LOGGER.warning("Playlists are not supported, "
"looking for the first video")
entries = list(all_media['entries'])
if len(entries) > 0:
selected_media = entries[0]
else:
_LOGGER.error("Playlist is empty")
raise MEDownloadException()
else:
selected_media = all_media
try:
media_info = ydl.process_ie_result(selected_media,
download=False)
except (ExtractorError, DownloadError):
# This exception will be logged by youtube-dl itself
raise MEDownloadException()
def stream_selector(query):
"""Find stream url that matches query."""
try:
format_selector = ydl.build_format_selector(query)
except (SyntaxError, ValueError, AttributeError) as ex:
_LOGGER.error(ex)
raise MEQueryException()
try:
requested_stream = next(format_selector(media_info))
except (KeyError, StopIteration):
_LOGGER.error("Could not extract stream for the query: %s",
query)
raise MEQueryException()
return requested_stream['url']
return stream_selector
def _extract_songs(self, ydl: youtube_dl.YoutubeDL, url: str):
info = ydl.extract_info(url, download=False)
if not info:
raise DownloadError('Not data could be retrieved')
if '_type' in info and info['_type'] == 'playlist':
entries = info['entries']
else:
entries = [info]
results = []
for info in entries:
result = (info['title'], info['url'])
results.append(result)
return results
def download(link, files_present, avg):
'''gets the playlist and Downloads the videos that i dont have'''
#url = 'https://www.youtube.com/watch?v=MCs5OvhV9S4'
url = 'https://www.youtube.com/playlist?list=PLwyG5wA5gIzjhW36BxGBoQwUZHnPDFux3'
ydl=YoutubeDL()
ydl.add_default_info_extractors()
playlist = ydl.extract_info(url, download=False)
for video in playlist['entries']:
import ipdb; ipdb.set_trace()
if video['title'] in files_present:
print ("Video #{} {} is present and will be ignored").format(video['playlist_index'], video['title'])
else:
print ("currently downloading: {}").format(video['title'])
ydl.download(video['webpage_url'])
class VideoLoader(BaseLoader):
CONTENT_TYPE = 'application/vnd.youtube-dl_formats+json'
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def load_resource(self, cdx, params):
load_url = cdx.get('load_url')
if not load_url:
return None
if params.get('content_type') != self.CONTENT_TYPE:
return None
if not self.ydl:
return None
info = self.ydl.extract_info(load_url)
info_buff = json.dumps(info)
info_buff = info_buff.encode('utf-8')
warc_headers = {}
schema, rest = load_url.split('://', 1)
target_url = 'metadata://' + rest
dt = timestamp_to_datetime(cdx['timestamp'])
warc_headers['WARC-Type'] = 'metadata'
warc_headers['WARC-Record-ID'] = self._make_warc_id()
warc_headers['WARC-Target-URI'] = target_url
warc_headers['WARC-Date'] = datetime_to_iso_date(dt)
warc_headers['Content-Type'] = self.CONTENT_TYPE
warc_headers['Content-Length'] = str(len(info_buff))
warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items())
return warc_headers, None, BytesIO(info_buff)
def test_ydl():
ydl = YoutubeDL(ydl_opt)
with ydl:
try:
info_dict = ydl.extract_info('https://www.youtube.com/watch?v=QwoghxwETng', download=False)
# print(info_dict.keys())
#print(info_dict['title'])
#print(ydl.list_formats(info_dict))
#for f in info_dict['formats']:
# filesize = f.get('filesize') if f.get('filesize') else f.get('filesize_approx')
# filesize2 = f['filesize'] if f['filesize'] else f['filesize_approx']
# print("%s, %s, %s, %s" % (f['format'], f['ext'], ydl.format_resolution(f), format_bytes(filesize2)))
#print(ydl.list_formats(info_dict))
except Exception as e:
print(e)
def __init__(self, dl_id, url):
self.dl_id = dl_id
self.url = url
self.start_time = datetime.datetime.utcnow()
self.download_proc = None
self.status_queue = multiprocessing.Queue()
self.status = {}
self.child_output = OutputWriter()
self.log = ""
info_downloader = YoutubeDL({"quiet": True})
info_downloader.add_default_info_extractors()
self.info = info_downloader.extract_info(self.url, download=False)
class VideoInfo(Command):
"""
Fetches video info. Currently we do title by default. Uses the
``youtube_dl`` library for all the information.
Example configuration that fetches titles from youtube, vimeo and
vine links::
{
"package": "mtj.jibberext.web.VideoInfo",
"alias": "video_info",
"kwargs": {
},
"commands": [
["(?P<url>http[s]?:\/\/((www.)?youtube.com|youtu.be)\/[\\w\\?=&\\-_]*|vimeo.com\/[0-9]*|vine.co\/v\/[\\w]*)",
"get_video_title"]
]
},
"""
def __init__(self):
if YoutubeDL is None:
raise RuntimeError('youtube-dl is not available')
self.ydl = YoutubeDL()
def extract_info(self, msg, match, bot, **kw):
if not match:
return {}
gd = match.groupdict()
url = gd.get('url')
if not url:
if not 'url' in gd:
logger.warning('URL match group may be missing in pattern?')
return {}
info = self.ydl.extract_info(url, download=False, process=False)
return info
def get_video_title(self, msg, match, bot, **kw):
info = self.extract_info(msg, match, bot, **kw)
return info.get('title')
class YoutubeDLWrapper(object): #pragma: no cover
""" YoutubeDL wrapper, inits youtubee-dl if it is available
"""
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
if not self.ydl:
return None
info = self.ydl.extract_info(url)
return info
def _parse(self):
if self.parseResults is None:
try:
inst = YoutubeDL({
"outtmpl": "%(title)s-%(id)s.%(ext)s",
"skip_download": True,
"quiet": True,
#"format": self.formats,
"verbose": True
})
#self.parseResults = inst.download([self.url])['entries'][0]
inst.get_info_extractor("Youtube")
inst.get_info_extractor("Vimeo")
self.parseResults = inst.extract_info(self.url, False)
except thirdparty_grabber.youtube_dl.utils.DownloadError as e:
if "This video does not exist" in e:
raise myexceptions.FetchingException("YouTube said: This video does not exist.", self.config.ERROR_404)
if "GEMA" in e:
raise myexceptions.FetchingException("BANNED BY GEMA", self.config.ERROR_GEMA)
raise myexceptions.FetchingException(e.message, self.config.ERROR_CUSTOMMESSAGE)
def get_stream_selector(self):
"""Return format selector for the media URL."""
from youtube_dl import YoutubeDL
from youtube_dl.utils import DownloadError, ExtractorError
ydl = YoutubeDL({'quiet': True, 'logger': _LOGGER})
try:
all_media = ydl.extract_info(self.get_media_url(), process=False)
except DownloadError:
# This exception will be logged by youtube-dl itself
raise MEDownloadException()
if 'entries' in all_media:
_LOGGER.warning(
"Playlists are not supported, looking for the first video")
entries = list(all_media['entries'])
if entries:
selected_media = entries[0]
else:
_LOGGER.error("Playlist is empty")
raise MEDownloadException()
else:
selected_media = all_media
def stream_selector(query):
"""Find stream URL that matches query."""
try:
ydl.params['format'] = query
requested_stream = ydl.process_ie_result(
selected_media, download=False)
except (ExtractorError, DownloadError):
_LOGGER.error(
"Could not extract stream for the query: %s", query)
raise MEQueryException()
return requested_stream['url']
return stream_selector
def _parse_url(url, callback):
download_tag = DownloadTag(url)
params = {"encoding": "utf8",
"format": "best",
"quiet": True,
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
try:
info_dict = ydl.extract_info(url, download=False)
except DownloadError as e:
_g_logger.exception(e)
download_tag.error = "Download Error. See error.txt for details"
else:
download_tag.ext = "." + info_dict["ext"]
download_tag.title = info_dict["title"]
download_tag.thumbnail_url = info_dict["thumbnail"]
download_tag.video_url = info_dict["url"]
wx.CallAfter(callback, download_tag)
def get_media_stream_url(media_url):
"""Extract stream URL from the media URL."""
from youtube_dl import YoutubeDL
from youtube_dl.utils import DownloadError, ExtractorError
ydl = YoutubeDL({'quiet': True, 'logger': _LOGGER})
try:
all_media_streams = ydl.extract_info(media_url, process=False)
except DownloadError:
# This exception will be logged by youtube-dl itself
raise YDException()
if 'entries' in all_media_streams:
_LOGGER.warning("Playlists are not supported, "
"looking for the first video")
try:
selected_stream = next(all_media_streams['entries'])
except StopIteration:
_LOGGER.error("Playlist is empty")
raise YDException()
else:
selected_stream = all_media_streams
try:
media_info = ydl.process_ie_result(selected_stream, download=False)
except (ExtractorError, DownloadError):
# This exception will be logged by youtube-dl itself
raise YDException()
format_selector = ydl.build_format_selector('best')
try:
best_quality_stream = next(format_selector(media_info))
except (KeyError, StopIteration):
best_quality_stream = media_info
return best_quality_stream['url']
def download(download):
"""Downloads the youtube video from the url
Args:
download: A DownloadRequest.
Returns:
A (file name, video title) tuple.
The file name is ONLY the file name, and does not include the file path.
"""
downloader = YoutubeDL()
downloader.add_default_info_extractors()
downloader.params['outtmpl'] = os.path.join(os.getcwd(),
'temp/%(id)s.%(ext)s')
downloader.params['verbose'] = True
downloader.params['cachedir'] = None
downloader.params['noplaylist'] = True
downloader.params['max_downloads'] = 1
if download.is_audio_only():
downloader.params['format'] = 'bestaudio'
else:
# We are only going to support downloading .mp4 videos.
downloader.params['format'] = 'mp4'
if download.get_force_mp4_filetype():
downloader.params['format'] = 'mp4'
info = downloader.extract_info(download.get_url())
file_name = downloader.prepare_filename(info)
file_name = file_name.encode('ascii', 'ignore')
title = info.get('title', os.path.basename(file_name))
return file_name, title
def __init__(self, url):
_youtube = YoutubeDL()
self._videourl = _youtube.extract_info(url, download=False)
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
is_playlist = any(k.startswith('playlist') for k in test_case)
test_cases = test_case.get(
'playlist', [] if is_playlist else [test_case])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
for tc in test_cases:
info_dict = tc.get('info_dict', {})
if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
if is_playlist and 'playlist' not in test_case:
params.setdefault('extract_flat', True)
params.setdefault('skip_download', True)
ydl = YoutubeDL(params, auto_init=False)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
expect_warnings(ydl, test_case.get('expected_warnings', []))
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
res_dict = None
def try_rm_tcs_files(tcs=None):
if tcs is None:
tcs = test_cases
for tc in tcs:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
# We're not using .download here sine that is just a shim
# for outside error handling, and returns the exit code
# instead of the result dict.
res_dict = ydl.extract_info(test_case['url'])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
if is_playlist:
self.assertEqual(res_dict['_type'], 'playlist')
self.assertTrue('entries' in res_dict)
expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
if 'playlist_mincount' in test_case:
assertGreaterEqual(
self,
len(res_dict['entries']),
test_case['playlist_mincount'],
'Expected at least %d in playlist %s, but got only %d' % (
test_case['playlist_mincount'], test_case['url'],
len(res_dict['entries'])))
if 'playlist_count' in test_case:
self.assertEqual(
len(res_dict['entries']),
test_case['playlist_count'],
'Expected %d entries in playlist %s, but got %d.' % (
test_case['playlist_count'],
test_case['url'],
len(res_dict['entries']),
))
if 'playlist_duration_sum' in test_case:
got_duration = sum(e['duration'] for e in res_dict['entries'])
self.assertEqual(
test_case['playlist_duration_sum'], got_duration)
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
expected_minsize = tc.get('file_minsize', 10000)
if expected_minsize is not None:
if params.get('test'):
expected_minsize = max(expected_minsize, 10000)
got_fsize = os.path.getsize(tc_filename)
assertGreaterEqual(
self, got_fsize, expected_minsize,
'Expected %s to be at least %s, but it\'s only %s ' %
(tc_filename, format_bytes(expected_minsize),
format_bytes(got_fsize)))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(
os.path.exists(info_json_fn),
'Missing info file %s' % info_json_fn)
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
if is_playlist and res_dict is not None and res_dict.get('entries'):
# Remove all other files that may have been extracted if the
# extractor returns full results even with extract_flat
res_tcs = [{'info_dict': e} for e in res_dict['entries']]
try_rm_tcs_files(res_tcs)
