def record_stream(channel, output_dir):
channels = get_channels("world", 2)
channel_url = "http://www.youtube.com/watch?v={}".format(channels[channel -
1])
print("Downloading from", channel_url)
ydl = YoutubeDL()
ydl.add_default_info_extractors()
info = ydl.extract_info(channel_url, download=False)
if info["uploader_id"] != "coachella":
print("Uploader is not coachella, stopping")
exit()
filename = "channel{}-{}.ts".format(channel, int(time.time()))
streamlink = Streamlink()
streamlink.set_loglevel("debug")
streamlink.set_logoutput(output_dir + "/" + filename + ".log")
streamlink.set_option("hls-live-edge", 9999999)
streamlink.set_option("hls-segment-attempts", 99)
streamlink.set_option("hls-segment-threads", 5)
streamlink.set_option("hls-segment-timeout", 9999)
streams = streamlink.streams(channel_url)
stream = streams["best"]
print("Found stream {}".format(stream.url))
print("Writing stream to {}".format(output_dir + "/" + filename))
with open(output_dir + "/" + filename,
'wb') as out_file, stream.open() as in_stream:
while True:
data = in_stream.read(1024)
if not data:
return
out_file.write(data)
def get_video_url(params):
"""Get video URL and start video player"""
url_selected = ''
all_datas_videos_quality = []
all_datas_videos_path = []
videos_html = utils.get_webcontent(params.video_url)
videos_soup = bs(videos_html, 'html.parser')
list_videos = videos_soup.find(
'ul', class_='nav nav-tabs').find_all('a')
for video in list_videos:
if '#video-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_jwplayer_id = video.get('data-jwplayer-id')
# Case mp4
if value_jwplayer_id != '':
list_streams = videos_soup.find_all(
'div', class_='jwplayer')
for stream in list_streams:
if stream.get('id') == value_jwplayer_id:
url = stream.get('data-source')
# Cas Yt
else:
video_id = re.compile(
'youtube.com/embed/(.*?)\?').findall(videos_html)[0]
url = resolver.get_stream_youtube(video_id, False)
all_datas_videos_path.append(url)
# Get link from FranceTV
elif '#ftv-player-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_ftvlayer_id = video.get('data-ftvplayer-id')
list_streams = videos_soup.find_all(
'iframe', class_='embed-responsive-item')
for stream in list_streams:
if stream.get('id') == value_ftvlayer_id:
url_id = stream.get('src')
ydl = YoutubeDL()
ydl.add_default_info_extractors()
with ydl:
result = ydl.extract_info(
url_id, download=False)
for format_video in result['formats']:
url = format_video['url']
all_datas_videos_path.append(url)
if len(all_datas_videos_quality) > 1:
seleted_item = common.sp.xbmcgui.Dialog().select(
common.GETTEXT('Choose video quality'),
all_datas_videos_quality)
if seleted_item == -1:
return ''
url_selected = all_datas_videos_path[seleted_item]
return url_selected
else:
return all_datas_videos_path[0]
class YoutubeDLWrapper(object):
""" Used to wrap youtubedl import, since youtubedl currently overrides
global HTMLParser.locatestarttagend regex with a different regex
that doesn't quite work.
This wrapper ensures that this regex is only set for YoutubeDL and unset
otherwise
"""
def __init__(self):
import HTMLParser as htmlparser
self.htmlparser = htmlparser
self.orig_tagregex = htmlparser.locatestarttagend
from youtube_dl import YoutubeDL as YoutubeDL
self.ydl_tagregex = htmlparser.locatestarttagend
htmlparser.locatestarttagend = self.orig_tagregex
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
info = None
try:
self.htmlparser.locatestarttagend = self.ydl_tagregex
info = self.ydl.extract_info(url)
finally:
self.htmlparser.locatestarttagend = self.orig_tagregex
return info
def YouPlay(vidUrl,isPlaylist=False):
VIDEO_URL = vidUrl
ydl_opt={"extract_flat":True,'quiet':True}
ydl = YoutubeDL(ydl_opt)
ydl.add_default_info_extractors()
if(not isPlaylist):
info = ydl.extract_info(VIDEO_URL,download=False)
title=info['title']
print(title)
url = info['formats'][0]['url']
instance = vlc.Instance('--no-video --quiet')
Vars.player = instance.media_player_new()
media = instance.media_new(url)
Vars.player.set_media(media)
Vars.player.play()
else:
ydl_opt={"extract_flat":True,'quiet':True,'playlistrandom':True}
ydl = YoutubeDL(ydl_opt)
info = ydl.extract_info(VIDEO_URL,download=False)
entries=info['entries']
Vars.playlistPlayer=PlaylistPlaying(Vars.player, entries)
Vars.playlistPlayer.start()
class ScramDLThread(threading.Thread):
def __init__(self, urls, addendum_params=None):
threading.Thread.__init__(self)
override_params = dict()
override_params['writeinfojson'] = False
override_params['test'] = False
override_params['format'] = 'mp4'
if addendum_params is not None:
override_params.update(addendum_params)
params = get_params(override_params)
self.scram_dl = YoutubeDL(params)
self.scram_dl.add_progress_hook(self.hook_progress)
self.scram_dl.add_default_info_extractors()
self.is_finished = False
self.urls = urls
def hook_progress(self, status):
if status['status'] == 'finished':
self.is_finished = True
else:
self.is_finished = False
def run(self):
self.scram_dl.download(self.urls)
def download(url, destination, logger):
opts = {
'format':
'm4a/bestaudio',
'outtmpl':
destination + '/%(title)s.%(ext)s',
'logger':
logger,
'progress_hooks': [dlProgress],
'postprocessors': [{
'key': 'MetadataFromTitle',
'titleformat': '%(artist)s - %(title)s'
}, {
'key': 'FFmpegMetadata'
}],
'ffmpeg_location':
resource_path('ffmpeg.exe')
}
ydl = YoutubeDL(opts)
ydl.add_default_info_extractors()
try:
ydl.download([url])
except:
app.enableButton("Download")
app.enableEntry("URL")
app.enableDirectoryEntry("destination")
app.queueFunction(writeOutput, sys.exc_info())
def process_item(self, item, spider):
fb_video_url = item.get('fb_video_url')
if fb_video_url:
ydl = YoutubeDL()
ydl.add_default_info_extractors()
ydl.extract_info(fb_video_url)
return item
class DownloadProcess(multiprocessing.Process):
"""
Actual download process which calls youtube-dl. You should never need to interact with this directly.
"""
def __init__(self, url, status_queue, output_writer):
super(DownloadProcess, self).__init__()
self.url = url
self.status_queue = status_queue
self.info = {}
sys.stdout = output_writer
sys.stderr = output_writer
self.downloader = YoutubeDL({"progress_hooks": [self.update_status]})
self.downloader.add_default_info_extractors()
def update_status(self, status):
self.status_queue.put(status)
def run(self):
self.info = self.downloader.extract_info(self.url)
def stop(self):
# Kill any child processes that may have spawned (e.g. ffmpeg)
import psutil, signal
s = psutil.Process()
for child in s.children(recursive=True):
child.send_signal(signal.SIGINT)
def run(event, context):
url = event['url']
ydl_opts = {
'outtmpl':
'/tmp/%(title)s-%(id)s.%(ext)s',
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'logger':
Logger(),
'progress_hooks': [processing_hook],
'ffmpeg_location':
'ffmpeg/',
}
ydl = YoutubeDL(ydl_opts)
ydl.add_default_info_extractors()
info = ydl.extract_info(url, True)
response = upload(info)
return {"url": response}
class PyJizzParser(object):
def __init__(self, model):
self.model = model
self.model.parser = self
self.ydl = YoutubeDL()
self.ydl.add_default_info_extractors()
def parseCategories(self):
c = PornHubCategoryParser(self.model)
c.run()
def parseCategoryPage(self, category, page = 1):
if page == 0 or page == 1:
url = self.model.categories_url[category]
else:
url = "{site}{page_url}{page}".format(
site = self.model.site,
page_url = self.model.p**n[category]['page_url'],
page = page)
print("page parser creating for page", page)
p = PornHubPageParser(self.model, url, category, page)
p.run()
print("page parser exit for page", page)
def getInfo(self, vkey):
info = self.ydl.extract_info('http://www.pornhub.com/view_video.php?viewkey={v}'.format(v = vkey), download=False)
return info
class YoutubeDLWrapper(object):
""" Used to wrap youtubedl import, since youtubedl currently overrides
global HTMLParser.locatestarttagend regex with a different regex
that doesn't quite work.
This wrapper ensures that this regex is only set for YoutubeDL and unset
otherwise
"""
def __init__(self):
import HTMLParser as htmlparser
self.htmlparser = htmlparser
self.orig_tagregex = htmlparser.locatestarttagend
from youtube_dl import YoutubeDL as YoutubeDL
self.ydl_tagregex = htmlparser.locatestarttagend
htmlparser.locatestarttagend = self.orig_tagregex
self.ydl = YoutubeDL(
dict(simulate=True, youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
info = None
try:
self.htmlparser.locatestarttagend = self.ydl_tagregex
info = self.ydl.extract_info(url)
finally:
self.htmlparser.locatestarttagend = self.orig_tagregex
return info
def process_url(self, source, global_opts):
"""Main method for processing urls
This method basically hands over the configuration to YoutubeDL and repeats
the step until every source in the configuration was read
"""
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url']
sourceDescription = source.get("description", "")
self._logsource(
"Processing source: '" + sourceDescription +
"' Url: '" + sourceUrl + "'", source)
# Merge local parameters with global ones
ydl.params = copy.copy(global_opts)
ydl.params.update(source)
prefix = ""
ydl.params['match_filter'] = (
None if 'match_filter' not in ydl.params or ydl.params['match_filter'] is None
else match_filter_func(ydl.params['match_filter']))
# Settings by commute tube over the standard settings, respect if the config sets them differently
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio/best" if 'format' not in self.config else self.config["format"]
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = self.download_archive
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
'%(uploader)s-%(title)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
elif not (ydl.params['outtmpl'].startswith(self.pathToDownloadFolder)):
self._logsource("Prefixing custom set outtmpl with '" + self.pathToDownloadFolder + "/" + prefix + "'", source)
ydl.params['outtmpl'] = self.pathToDownloadFolder + "/" + prefix + \
ydl.params['outtmpl']
if self.debug:
self._logsource(
"All downloads will be simulated since this is debug mode", source)
ydl.params['simulate'] = True
ydl.download([source['url']])
def sys_main():
ydl = YoutubeDL()
ydl.add_default_info_extractors()
info = ydl.extract_info('https://www.youtube.com/watch?v=-tzvebu6U08',
download=False)
for x in info['formats']:
print(x['format'], x['url'])
return 0
def get_info(url):
ydl = YoutubeDL({
'forceurl': True,
'quiet': True
})
ydl.add_default_info_extractors()
return ydl.extract_info(url, download=False)
def get_youtube_info(url):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
try:
info = ydl.extract_info(url, download=False)
except:
return None
return info
def get_youtube_info(url):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
try:
info = ydl.extract_info(url, download=False)
except:
return None
return info
def youtube_downloader(url):
ydl_options = {'outtmpl': video_files_path + '%(title)s.%(ext)s'}
ydl = YoutubeDL(ydl_options)
ydl.add_default_info_extractors()
info = ydl.extract_info(
url=url,
download=True,
)
return ydl.prepare_filename(info)
def get_video_url(params):
"""Get video URL and start video player"""
url_selected = ''
all_datas_videos_quality = []
all_datas_videos_path = []
videos_html = utils.get_webcontent(params.video_url)
videos_soup = bs(videos_html, 'html.parser')
list_videos = videos_soup.find('ul', class_='nav nav-tabs').find_all('a')
for video in list_videos:
if '#video-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_jwplayer_id = video.get('data-jwplayer-id')
# Case mp4
if value_jwplayer_id != '':
list_streams = videos_soup.find_all('div', class_='jwplayer')
for stream in list_streams:
if stream.get('id') == value_jwplayer_id:
url = stream.get('data-source')
# Cas Yt
else:
video_id = re.compile('youtube.com/embed/(.*?)\?').findall(
videos_html)[0]
url = resolver.get_stream_youtube(video_id, False)
all_datas_videos_path.append(url)
# Get link from FranceTV
elif '#ftv-player-' in video.get('href'):
# Find a better solution to strip
all_datas_videos_quality.append(video.get_text().strip())
# Get link
value_ftvlayer_id = video.get('data-ftvplayer-id')
list_streams = videos_soup.find_all('iframe',
class_='embed-responsive-item')
for stream in list_streams:
if stream.get('id') == value_ftvlayer_id:
url_id = stream.get('src')
ydl = YoutubeDL()
ydl.add_default_info_extractors()
with ydl:
result = ydl.extract_info(url_id, download=False)
for format_video in result['formats']:
url = format_video['url']
all_datas_videos_path.append(url)
if len(all_datas_videos_quality) > 1:
seleted_item = common.sp.xbmcgui.Dialog().select(
common.GETTEXT('Choose video quality'), all_datas_videos_quality)
if seleted_item == -1:
return ''
url_selected = all_datas_videos_path[seleted_item]
return url_selected
else:
return all_datas_videos_path[0]
def run(argument):
if argument['type'] == 'magnet':
openWith = utils.getSetting('openMagnetWith')
logger.info('Sharing magnet with %s' % openWith)
if openWith == 'Elementum':
utils.callPlugin('plugin://plugin.video.elementum/playuri?uri=' +
argument['data'] + ')')
elif openWith == 'Torrenter V2':
utils.callPlugin(
'plugin://plugin.video.torrenter/?action=playSTRM&url=' +
argument['data'] + ')')
elif openWith == 'Quasar':
utils.callPlugin('plugin://plugin.video.quasar/playuri?uri=' +
argument['data'] + ')')
elif argument['type'] == 'unresolvedurl':
url = urllib.unquote(argument['data'])
logger.info('Trying to resolve with urlresolver: %s' % url)
stream_url = urlresolver.HostedMediaFile(url=url).resolve()
if not stream_url:
if have_youtube_dl == "false":
logger.error("Url not resolved and no YoutubeDL: %s" % url)
else:
logger.info('Trying to resolve with YoutubeDL: %s' % url)
YoutubeDL_resolver = YoutubeDL({'format': 'best'})
YoutubeDL_resolver.add_default_info_extractors()
result = YoutubeDL_resolver.extract_info(url, download=False)
if "url" in result:
stream_url = result["url"]
logger.info("Url resolved by YoutubeDL: %s" % result)
else:
logger.error("Url not resolved: %s" % url)
else:
logger.info("Url resolved by urlResolver: %s" % stream_url)
if stream_url:
if (not 'queue' in argument) or (argument['queue'] == "false") or (
not xbmc.Player().isPlaying()):
logger.info('Playing resolved url: %s' % stream_url)
xbmc.Player().play(stream_url)
else:
if xbmc.Player().isPlayingAudio():
logger.info('Queuing to music resolved url: %s' %
stream_url)
xbmc.PlayList(xbmc.PLAYLIST_MUSIC).add(stream_url)
else:
logger.info('Queuing to video resolved url: %s' %
stream_url)
xbmc.PlayList(xbmc.PLAYLIST_VIDEO).add(stream_url)
else:
dialog = xbmcgui.Dialog()
dialog.notification(utils.ADDON_NAME, utils.translation(32006),
xbmcgui.NOTIFICATION_INFO, 5000)
def extractMeta(link_out):
global gen_info
gen_info = []
ydl = YoutubeDL()
ydl.add_default_info_extractors()
for link in link_out:
meta = ydl.extract_info(link, download=False)
for formats in meta['formats']:
if formats['format_id'] == '18':
gen_info.append([
meta['id'], meta['title'], meta['thumbnail'],
formats['url']
])
def get_stream_youtube(video_id, isDownloadVideo):
url_youtube = URL_YOUTUBE % video_id
if isDownloadVideo == True:
return url_youtube
ydl = YoutubeDL()
ydl.add_default_info_extractors()
with ydl:
result = ydl.extract_info(url_youtube, download=False)
for format_video in result['formats']:
url_live = format_video['url']
return url_live
class VideoLoader(BaseLoader):
CONTENT_TYPE = 'application/vnd.youtube-dl_formats+json'
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(
dict(simulate=True,
quiet=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def load_resource(self, cdx, params):
load_url = cdx.get('load_url')
if not load_url:
return None
if params.get('content_type') != self.CONTENT_TYPE:
return None
if not self.ydl:
return None
info = self.ydl.extract_info(load_url)
info_buff = json.dumps(info)
info_buff = info_buff.encode('utf-8')
warc_headers = {}
schema, rest = load_url.split('://', 1)
target_url = 'metadata://' + rest
dt = timestamp_to_datetime(cdx['timestamp'])
warc_headers['WARC-Type'] = 'metadata'
warc_headers['WARC-Record-ID'] = self._make_warc_id()
warc_headers['WARC-Target-URI'] = target_url
warc_headers['WARC-Date'] = datetime_to_iso_date(dt)
warc_headers['Content-Type'] = self.CONTENT_TYPE
warc_headers['Content-Length'] = str(len(info_buff))
warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items())
return warc_headers, None, BytesIO(info_buff)
def download(link, files_present, avg):
'''gets the playlist and Downloads the videos that i dont have'''
#url = 'https://www.youtube.com/watch?v=MCs5OvhV9S4'
url = 'https://www.youtube.com/playlist?list=PLwyG5wA5gIzjhW36BxGBoQwUZHnPDFux3'
ydl=YoutubeDL()
ydl.add_default_info_extractors()
playlist = ydl.extract_info(url, download=False)
for video in playlist['entries']:
import ipdb; ipdb.set_trace()
if video['title'] in files_present:
print ("Video #{} {} is present and will be ignored").format(video['playlist_index'], video['title'])
else:
print ("currently downloading: {}").format(video['title'])
ydl.download(video['webpage_url'])
def download_youtube(url,filepath,params=None):
tmp_filepath = compat_str(filepath)
print "download to",tmp_filepath
params = params or settings.youtube_params
params.update({"outtmpl":tmp_filepath,"daterange":DateRange(None,None)})
y = YoutubeDL(params)
#y = YoutubeDL({"format":"18/34/35/5/17","outtmpl":filepath})
#y.print_debug_header()
y.add_default_info_extractors()
y.add_post_processor(FFmpegExtractAudioPP(preferredcodec="m4a",preferredquality=5, nopostoverwrites=False))
value = y.download([url])
#cmd = 'youtube-dl {url} --extract-audio --audio-format wav -o {filepath}'.format(url=url,filepath=filepath)
#print cmd
#result = subprocess.call(cmd,shell=True)
#print result
return True
class VideoLoader(BaseLoader):
CONTENT_TYPE = 'application/vnd.youtube-dl_formats+json'
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def load_resource(self, cdx, params):
load_url = cdx.get('load_url')
if not load_url:
return None
if params.get('content_type') != self.CONTENT_TYPE:
return None
if not self.ydl:
return None
info = self.ydl.extract_info(load_url)
info_buff = json.dumps(info)
info_buff = info_buff.encode('utf-8')
warc_headers = {}
schema, rest = load_url.split('://', 1)
target_url = 'metadata://' + rest
dt = timestamp_to_datetime(cdx['timestamp'])
warc_headers['WARC-Type'] = 'metadata'
warc_headers['WARC-Record-ID'] = self._make_warc_id()
warc_headers['WARC-Target-URI'] = target_url
warc_headers['WARC-Date'] = datetime_to_iso_date(dt)
warc_headers['Content-Type'] = self.CONTENT_TYPE
warc_headers['Content-Length'] = str(len(info_buff))
warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items())
return warc_headers, None, BytesIO(info_buff)
def _download_restricted(url, filename, age):
""" Returns true iff the file has been downloaded """
params = {
'age_limit': age,
'skip_download': True,
'writeinfojson': True,
"outtmpl": "%(id)s.%(ext)s",
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
json_filename = filename + '.info.json'
try_rm(json_filename)
ydl.download([url])
res = os.path.exists(json_filename)
try_rm(json_filename)
return res
def on_task_output(self, task, config):
import youtube_dl.YoutubeDL
from youtube_dl.utils import ExtractorError
class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs):
self.to_stderr = self.to_screen
self.processed_info_dicts = []
super(YoutubeDL, self).__init__(*args, **kwargs)
def report_warning(self, message):
raise ExtractorError(message)
def process_info(self, info_dict):
self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict)
for entry in task.accepted:
if task.options.test:
log.info('Would download %s' % entry['title'])
else:
try:
outtmpl = entry.render(config['path']) + '/' + pathscrub(entry.render(config['template']) + '.%(ext)s', filename=True)
log.info("Output file: %s" % outtmpl)
except RenderError as e:
log.error('Error setting output file: %s' % e)
entry.fail('Error setting output file: %s' % e)
params = {'quiet': True, 'outtmpl': outtmpl}
if 'username' in config and 'password' in config:
params.update({'username': config['username'], 'password': config['password']})
elif 'username' in config or 'password' in config:
log.error('Both username and password is required')
if 'videopassword' in config:
params.update({'videopassword': config['videopassword']})
if 'title' in config:
params.update({'title': config['title']})
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
log.info('Downloading %s' % entry['title'])
try:
ydl.download([entry['url']])
except ExtractorError as e:
log.error('Youtube-DL was unable to download the video. Error message %s' % e.message)
entry.fail('Youtube-DL was unable to download the video. Error message %s' % e.message)
except Exception as e:
log.error('Youtube-DL failed. Error message %s' % e.message)
entry.fail('Youtube-DL failed. Error message %s' % e.message)
def _download_restricted(url, filename, age):
""" Returns true iff the file has been downloaded """
params = {
'age_limit': age,
'skip_download': True,
'writeinfojson': True,
"outtmpl": "%(id)s.%(ext)s",
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
json_filename = filename + '.info.json'
try_rm(json_filename)
ydl.download([url])
res = os.path.exists(json_filename)
try_rm(json_filename)
return res
def __init__(self, dl_id, url):
self.dl_id = dl_id
self.url = url
self.start_time = datetime.datetime.utcnow()
self.download_proc = None
self.status_queue = multiprocessing.Queue()
self.status = {}
self.child_output = OutputWriter()
self.log = ""
info_downloader = YoutubeDL({"quiet": True})
info_downloader.add_default_info_extractors()
self.info = info_downloader.extract_info(self.url, download=False)
def processUrl(self, source):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url'].decode()
sourceDescription = ""
if 'description' in source:
sourceDescription = source['description'].decode()
self.log.info(
"Processing source: '" + sourceDescription
+ "' Url: '" + sourceUrl + "'")
ydl.params = source
prefix = ""
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio"
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = "already_downloaded.txt"
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
u'%(uploader)s-%(title)s-%(id)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
if self.debug is True:
self.log.debug(
"All downloads will be simulated since this is debug mode")
ydl.params['simulate'] = True
ydl.download([source['url']])
def get_videos_from_playlist(self):
playlist = self.get_YT_playlists()
request = self.youtube_client.playlistItems().list(
part="snippet", playlistId=self.playlists[playlist], maxResults=50)
response = request.execute()
ydl = YoutubeDL()
ydl.add_default_info_extractors()
for item in response["items"]:
info = ydl.extract_info('http://www.youtube.com/watch?v=' +
item["snippet"]["resourceId"]["videoId"],
download=False)
uri = self.get_Spotify_song_uri(info["track"], info["artist"])
if uri is None:
continue
self.uris.append(uri)
def resolve_with_youtube_dl(url, parameters, action):
youtube_dl_resolver = YoutubeDL(parameters)
youtube_dl_resolver.add_default_info_extractors()
try:
result = youtube_dl_resolver.extract_info(url, download=False)
if result is None:
result = {}
except Exception as e:
logger.error(u'Error with YoutubeDL: %s' % e)
result = {}
logger.info(u'YoutubeDL full result: %s' % result)
if 'entries' in result:
logger.info(u'Playlist resolved by YoutubeDL: %s items' % len(result['entries']))
item_list = []
for entry in result['entries']:
if entry is not None and 'url' in entry:
item_list.append(entry)
logger.info(u'Media found: %s' % entry['url'])
if len(item_list) > 0:
utils.play_items(item_list, action)
return True
else:
logger.info(u'No playable urls in the playlist')
if 'url' in result:
logger.info(u'Url resolved by YoutubeDL: %s' % result['url'])
utils.play_url(result['url'], action, result)
return True
if 'requested_formats' in result:
if have_adaptive_plugin:
logger.info(u'Adaptive plugin enabled looking for dash content')
for entry in result['requested_formats']:
if 'container' in entry and 'manifest_url' in entry:
if 'dash' in entry['container']:
logger.info(u'Url resolved by YoutubeDL: %s' % entry['manifest_url'])
utils.play_url(entry['manifest_url'], action, result, True)
return True
for entry in result['requested_formats']:
if 'protocol' in entry and 'manifest_url' in entry:
if 'm3u8' in entry['protocol']:
logger.info(u'Url resolved by YoutubeDL: %s' % entry['manifest_url'])
utils.play_url(entry['manifest_url'], action, result)
return True
return False
def processUrl(self, source):
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url'].decode()
sourceDescription = ""
if 'description' in source:
sourceDescription = source['description'].decode()
self.log.info("Processing source: '" + sourceDescription + "' Url: '" +
sourceUrl + "'")
ydl.params = source
prefix = ""
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params['format'] = "bestvideo+bestaudio"
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = "already_downloaded.txt"
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
u'%(uploader)s-%(title)s-%(id)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
if self.debug is True:
self.log.debug(
"All downloads will be simulated since this is debug mode")
ydl.params['simulate'] = True
ydl.download([source['url']])
class YoutubeDLWrapper(object): #pragma: no cover
""" YoutubeDL wrapper, inits youtubee-dl if it is available
"""
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(dict(simulate=True,
youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
if not self.ydl:
return None
info = self.ydl.extract_info(url)
return info
class YoutubeDLWrapper(object): #pragma: no cover
""" YoutubeDL wrapper, inits youtubee-dl if it is available
"""
def __init__(self):
try:
from youtube_dl import YoutubeDL as YoutubeDL
except ImportError:
self.ydl = None
return
self.ydl = YoutubeDL(
dict(simulate=True, youtube_include_dash_manifest=False))
self.ydl.add_default_info_extractors()
def extract_info(self, url):
if not self.ydl:
return None
info = self.ydl.extract_info(url)
return info
def indir(self):
os.getcwd()
os.chdir(
os.getcwd()
) # Videoların programın olduğu dizine kaydetmesini bu iki satır sağlıyor.
if self.veri.text().startswith("https://www.youtube.com/"):
ydl_ayarlar = {'noplaylist': True, 'progress_hooks': [self.hook]}
QMessageBox.information(
self, "Videonuz işlenmeye hazır",
"İşleme süreci indirdiğiniz videonun boyutuna bağlıdır bekleme süreniz de buna göre "
"değişebilir.İşleme süreci bittiğinde bar dolmaya başlayacak ve video indirilecektir."
)
self.setWindowTitle("Videonuz İşleniyor....")
ydl = YoutubeDL(ydl_ayarlar)
ydl.add_default_info_extractors()
info = ydl.extract_info(self.veri.text(), download=True)
self.setWindowTitle("Videonuz İndiriliyor....")
self.buton.setEnabled(False)
self.bariSekillendir()
indirilen_oge = info["title"]
self.liste.insertItem(0, indirilen_oge)
self.setWindowTitle("Youtube [HD] MP4 İndirici | Cankat Özçakır")
QMessageBox.information(
self, "İndirme Başarılı!",
"İndirme işlemini başarıyla gerçekleştirdiniz.")
self.buton.setEnabled(True)
elif not self.veri.text():
self.bilgilendirme.setText("Herhangi bir adres belirtmediniz.")
QTimer.singleShot(2000, self.bilgilendirmeYazisi)
else:
self.bilgilendirme.setText(
"Yalnızca Youtube'dan video indirebilirsiniz.")
QTimer.singleShot(2000, self.bilgilendirmeYazisi)
def download_video(channel: dict, video: dict) -> pathlib.Path:
"""
Download a video (and associated posters/etc) to it's channel's directory.
:param channel:
:param video: A YoutubeDL info entry dictionary
:return:
"""
# YoutubeDL expects specific options, add onto the default options
config = get_downloader_config()
options = dict(config)
directory = get_absolute_media_path(channel['directory'])
options['outtmpl'] = f'{directory}/{config["file_name_format"]}'
ydl = YoutubeDL(options)
ydl.add_default_info_extractors()
source_id = video['id']
url = f'https://www.youtube.com/watch?v={source_id}'
entry = ydl.extract_info(url, download=True, process=True)
final_filename = ydl.prepare_filename(entry)
final_filename = pathlib.Path(final_filename)
return final_filename
def _parse_url(url, callback):
download_tag = DownloadTag(url)
params = {"encoding": "utf8",
"format": "best",
"quiet": True,
}
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
try:
info_dict = ydl.extract_info(url, download=False)
except DownloadError as e:
_g_logger.exception(e)
download_tag.error = "Download Error. See error.txt for details"
else:
download_tag.ext = "." + info_dict["ext"]
download_tag.title = info_dict["title"]
download_tag.thumbnail_url = info_dict["thumbnail"]
download_tag.video_url = info_dict["url"]
wx.CallAfter(callback, download_tag)
def lambda_handler(event, context):
url = event['url'] ### Need to be changed for Api Gateway POST request
ydl_opts = {
'outtmpl':
'/tmp/%(title)s-%(id)s.%(ext)s',
'format':
'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'logger':
MyLogger(),
'progress_hooks': [my_hook],
'ffmpeg_location':
'ffmpeg/',
}
ydl = YoutubeDL(ydl_opts)
ydl.add_default_info_extractors()
info = ydl.extract_info(url, True)
response = upload(info)
return {"url": response}
def download(download):
"""Downloads the youtube video from the url
Args:
download: A DownloadRequest.
Returns:
A (file name, video title) tuple.
The file name is ONLY the file name, and does not include the file path.
"""
downloader = YoutubeDL()
downloader.add_default_info_extractors()
downloader.params['outtmpl'] = os.path.join(os.getcwd(),
'temp/%(id)s.%(ext)s')
downloader.params['verbose'] = True
downloader.params['cachedir'] = None
downloader.params['noplaylist'] = True
downloader.params['max_downloads'] = 1
if download.is_audio_only():
downloader.params['format'] = 'bestaudio'
else:
# We are only going to support downloading .mp4 videos.
downloader.params['format'] = 'mp4'
if download.get_force_mp4_filetype():
downloader.params['format'] = 'mp4'
info = downloader.extract_info(download.get_url())
file_name = downloader.prepare_filename(info)
file_name = file_name.encode('ascii', 'ignore')
title = info.get('title', os.path.basename(file_name))
return file_name, title
class WhydScraper():
""" All your tracks are belong to us. """
def __init__(self):
self.ydl = YoutubeDL(YDL_PARAMS)
self.ydl.add_default_info_extractors()
def _make_soup(self, url):
""" Helper to make a BeautifulSoup object.
:param url: URL to scrap.
"""
try:
return BeautifulSoup(requests.get(url).text)
except ValueError:
logging.warning('The URL you provided is not valid : "%s"' % url)
return None
def scrap_playlists(self, username):
""" Scrap user playlists from his profile page.
:param username: User name (slug) OR id (/u/xxxx).
"""
soup = self._make_soup('{root_url}/{username}/playlists'.format(
root_url=ROOT_URL, username=username))
pl_tags = soup.find_all('li', class_='playlist')
return [li.a['href'].split('/')[-1] for li in pl_tags]
def get_playlist_by_id(self, username, pl_id, limit=PLAYLIST_LIMIT):
""" Get tracklist by Username + ID."""
res = requests.get(
'{root_url}/{username}/playlist/{pl_id}?format=json&limit={limit}'.format(
root_url=ROOT_URL, username=username, pl_id=pl_id, limit=limit))
tracks = self._format_json(res.json())
return tracks
def get_playlist_by_url(self, pl_url, limit=PLAYLIST_LIMIT):
""" Get tracklist by URL. """
res = requests.get('{url}?format=json&limit={limit}'.format(
url=pl_url, limit=limit))
tracks = self._format_json(res.json())
return tracks
def _format_json(self, json):
""" Parse JSON and return formated dict. """
# format urls
for track in json:
for e in PROVIDERS:
if PROVIDERS[e]['prefix'] in track['eId']:
track['eId'] = track['eId'].replace(
PROVIDERS[e]['prefix'],
PROVIDERS[e]['root_url']).split('#')[0]
try:
playlist = track['pl']['name']
except KeyError:
playlist = 'None'
return [{'playlist': playlist,
'name': track['name'],
'url': track['eId']} for track in json]
def get_user_stream(self, username, limit='20'):
""" Get user tracklist from its added tracks stream.
:param username: User nickname (slug) OR id (/u/xxxx).
:param limit: Maximum number of tracks to retrieve.
"""
res = requests.get(
'{root_url}/{username}?format=json&limit={limit}'.format(
root_url=ROOT_URL, username=username, limit=limit))
tracks = self._format_json(res.json())
return tracks
def get_user_likes(self, username):
""" Get user tracklist from its liked tracks stream.
:param username: User nickname (slug) OR id (/u/xxxx).
"""
res = requests.get(
'{root_url}/{username}/likes?format=json'.format(
root_url=ROOT_URL, username=username))
tracks = self._format_json(res.json())
return tracks
def download(self, url, outdir, dry_run=False):
""" Download the audio version of a given track url.
:param url: Track URL.
:param outdir: Output directory.
"""
if dry_run:
print(url)
return
# output root folder
try:
os.mkdir('output')
except OSError:
pass
# playlist specific folder
try:
os.mkdir('output/%s' % outdir)
logging.info('output/%s created.' % outdir)
except OSError:
logging.info('output directory already exists, skipping creation.')
self.ydl.params['outtmpl'] = 'output/' + outdir + '/%(title)s.%(ext)s'
try:
self.ydl.extract_info(url, download=True)
except DownloadError as e:
logging.info(e)
return
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie._WORKING:
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case and not test_case['file']:
print_skipping('No output file specified')
return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
params = self.parameters.copy()
params.update(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.fd.add_progress_hook(_hook)
test_cases = test_case.get('playlist', [test_case])
for tc in test_cases:
_try_rm(tc['file'])
_try_rm(tc['file'] + '.part')
_try_rm(tc['file'] + '.info.json')
try:
for retry in range(1, RETRIES + 1):
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
if retry == RETRIES: raise
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
raise
print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
else:
break
for tc in test_cases:
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
self.assertTrue(tc['file'] in finished_hook_called)
self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
if 'md5' in tc:
md5_for_file = _file_md5(tc['file'])
self.assertEqual(md5_for_file, tc['md5'])
with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected, compat_str) and expected.startswith('md5:'):
got = 'md5:' + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(expected, got,
u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n')
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
finally:
for tc in test_cases:
_try_rm(tc['file'])
_try_rm(tc['file'] + '.part')
_try_rm(tc['file'] + '.info.json')
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case["name"])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get("add_ie", [])]
def print_skipping(reason):
print("Skipping %s: %s" % (test_case["name"], reason))
if not ie.working():
print_skipping("IE marked as not _WORKING")
return
if "playlist" not in test_case:
info_dict = test_case.get("info_dict", {})
if not test_case.get("file") and not (info_dict.get("id") and info_dict.get("ext")):
print_skipping(
'The output file cannot be know, the "file" ' "key is missing or the info_dict is incomplete"
)
return
if "skip" in test_case:
print_skipping(test_case["skip"])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u"test depends on %sIE, marked as not WORKING" % other_ie.ie_key())
return
params = get_params(test_case.get("params", {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status["status"] == "finished":
finished_hook_called.add(status["filename"])
ydl.fd.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get("file") or ydl.prepare_filename(tc.get("info_dict", {}))
test_cases = test_case.get("playlist", [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + ".part")
try_rm(os.path.splitext(tc_filename)[0] + ".info.json")
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case["url"]])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or (
err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503
):
raise
if try_num == RETRIES:
report_warning(u"Failed due to network errors, skipping...")
return
print("Retrying: {0} failed tries\n\n##########\n\n".format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get("params", {}).get("skip_download", False):
self.assertTrue(os.path.exists(tc_filename), msg="Missing file " + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + ".info.json"
self.assertTrue(os.path.exists(info_json_fn))
if "md5" in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc["md5"])
with io.open(info_json_fn, encoding="utf-8") as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get("info_dict", {}).items():
if isinstance(expected, compat_str) and expected.startswith("md5:"):
got = "md5:" + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(
expected, got, u"invalid value for field %s, expected %r, got %r" % (info_field, expected, got)
)
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict(
(key, value if not isinstance(value, compat_str) or len(value) < 250 else "md5:" + md5(value))
for key, value in info_dict.items()
if value and key in ("title", "description", "uploader", "upload_date", "uploader_id", "location")
)
if not all(key in tc.get("info_dict", {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(
u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u"\n"
)
# Check for the presence of mandatory fields
for key in ("id", "url", "title", "ext"):
self.assertTrue(key in info_dict.keys() and info_dict[key])
# Check for mandatory fields that are automatically set by YoutubeDL
for key in ["webpage_url", "extractor", "extractor_key"]:
self.assertTrue(info_dict.get(key), u"Missing field: %s" % key)
finally:
try_rm_tcs_files()
class NBC(object):
def __init__(self):
log('__init__')
self.cache = SimpleCache()
self.ydl = YoutubeDL()
def openURL(self, url):
log('openURL, url = ' + str(url))
try:
cacheResponce = self.cache.get(ADDON_NAME +
'.openURL, url = %s' % url)
if not cacheResponce:
request = urllib2.Request(url)
responce = urllib2.urlopen(request, timeout=TIMEOUT).read()
self.cache.set(ADDON_NAME + '.openURL, url = %s' % url,
responce,
expiration=datetime.timedelta(hours=1))
return self.cache.get(ADDON_NAME + '.openURL, url = %s' % url)
except urllib2.URLError as e:
log("openURL Failed! " + str(e), xbmc.LOGERROR)
except socket.timeout as e:
log("openURL Failed! " + str(e), xbmc.LOGERROR)
except Exception as e:
log("openURL Failed! " + str(e), xbmc.LOGERROR)
xbmcgui.Dialog().notification(ADDON_NAME, LANGUAGE(30001), ICON,
4000)
return ''
def buildMenu(self, items):
for item in items:
self.addDir(*item)
self.addYoutube("Browse Youtube",
'plugin://plugin.video.youtube/user/NBC/')
def browseEpisodes(self, url=None):
log('browseEpisodes')
if url is None:
url = VIDEO_URL + FILTER % ('type', 'Full%20Episode')
items = json.loads(self.openURL(url))
if items and 'data' in items:
for item in items['data']:
path = item['attributes']['fullUrl']
aired = str(item['attributes']['airdate']).split('T')[0]
duration = int(item['attributes']['runTime'])
plot = item['attributes']['description']
title = item['attributes']['title']
showTitle = ''
for show in item['attributes']['categories']:
if show.startswith('Series'):
showTitle = show.split('Series/')[1]
break
try:
episodeNumber = int(item['attributes']['episodeNumber'])
except:
episodeNumber = 0
try:
seasonNumber = int(item['attributes']['seasonNumber'])
except:
seasonNumber = 0
try:
thumb = ICON
for image in items['included']:
if image['id'] == item['relationships']['image'][
'data']['id']:
thumb = BASE_URL + image['attributes']['path']
break
except:
thumb = ICON
seinfo = ('S' + ('0' if seasonNumber < 10 else '') +
str(seasonNumber) + 'E' +
('0' if episodeNumber < 10 else '') +
str(episodeNumber))
label = '%s - %s' % (
showTitle, title
) if seasonNumber + episodeNumber == 0 else '%s - %s - %s' % (
showTitle, seinfo, title)
infoLabels = {
"mediatype": "episodes",
"label": label,
"title": label,
"TVShowTitle": showTitle,
"plot": plot,
"aired": aired,
"duration": duration,
"season": seasonNumber,
"episode": episodeNumber
}
infoArt = {
"thumb": thumb,
"poster": thumb,
"fanart": FANART,
"icon": ICON,
"logo": ICON
}
self.addLink(label, path, 9, infoLabels, infoArt,
len(items['data']))
try:
next_page = items['links']['next']
except:
next_page = None
if next_page:
self.addDir('>> Next', next_page, 1)
def browseShows(self, url=None):
log('browseShows')
if url is None:
url = SHOWS_URL
items = json.loads(self.openURL(url))
if items and 'data' in items:
for item in items['data']:
showTitle = item['attributes']['shortTitle']
plot = (item['attributes']['shortDescription']
or showTitle).replace('<p>', '').replace('</p>', '')
path = VIDEO_URL + FILTER % ('show', item['id'])
vidID = item['relationships']['aggregates']['data']['id']
try:
thumb = ICON
for image in items['included']:
if image['id'] == item['relationships']['image'][
'data']['id']:
thumb = BASE_URL + image['attributes']['path']
break
except:
thumb = ICON
myURL = json.dumps({"url": path, "vidID": vidID})
infoLabels = {
"mediatype": "tvshows",
"label": showTitle,
"title": showTitle,
"TVShowTitle": showTitle,
"plot": plot
}
infoArt = {
"thumb": thumb,
"poster": thumb,
"fanart": FANART,
"icon": ICON,
"logo": ICON
}
self.addDir(showTitle, myURL, 0, infoLabels, infoArt)
try:
next_page = items['links']['next']
except:
next_page = None
if next_page:
self.addDir('>> Next', next_page, 2)
def resolveURL(self, url):
log('resolveURL')
myURL = json.loads(url)
items = json.loads(self.openURL(SHOW_URL % myURL['vidID']))
if items and 'data' in items:
for item in items['data']['attributes']['videoTypes']:
self.browseEpisodes(myURL['url'] + FILTER %
('type', urllib2.quote(item)))
def playVideo(self, name, url, liz=None):
log('playVideo')
self.ydl.add_default_info_extractors()
with self.ydl:
result = self.ydl.extract_info(url, download=False)
url = result['manifest_url']
liz = xbmcgui.ListItem(name, path=url)
xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, liz)
def addYoutube(self, name, url):
liz = xbmcgui.ListItem(name)
liz.setProperty('IsPlayable', 'false')
liz.setInfo(type="Video", infoLabels={"label": name, "title": name})
liz.setArt({'thumb': ICON, 'fanart': FANART})
xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]),
url=url,
listitem=liz,
isFolder=True)
def addLink(self, name, u, mode, infoList=False, infoArt=False, total=0):
name = name.encode("utf-8")
log('addLink, name = ' + name)
liz = xbmcgui.ListItem(name)
liz.setProperty('IsPlayable', 'true')
if infoList == False:
liz.setInfo(type="Video",
infoLabels={
"mediatype": "video",
"label": name,
"title": name,
"genre": "News"
})
else:
liz.setInfo(type="Video", infoLabels=infoList)
if infoArt == False:
liz.setArt({'thumb': ICON, 'fanart': FANART})
else:
liz.setArt(infoArt)
u = sys.argv[0] + "?url=" + urllib.quote_plus(u) + "&mode=" + str(
mode) + "&name=" + urllib.quote_plus(name)
xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]),
url=u,
listitem=liz,
totalItems=total)
def addDir(self, name, u, mode, infoList=False, infoArt=False):
name = name.encode("utf-8")
log('addDir, name = ' + name)
liz = xbmcgui.ListItem(name)
liz.setProperty('IsPlayable', 'false')
if infoList == False:
liz.setInfo(type="Video",
infoLabels={
"mediatype": "video",
"label": name,
"title": name,
"genre": "News"
})
else:
liz.setInfo(type="Video", infoLabels=infoList)
if infoArt == False:
liz.setArt({'thumb': ICON, 'fanart': FANART})
else:
liz.setArt(infoArt)
u = sys.argv[0] + "?url=" + urllib.quote_plus(u) + "&mode=" + str(
mode) + "&name=" + urllib.quote_plus(name)
xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]),
url=u,
listitem=liz,
isFolder=True)
def play_url(): # this only plays http urls for now, torrents soon.
global title
url = request.args.get('url') # grab url from /play?url=*
if not url.startswith('http'): # in case the user forgot it
log('url missing http/wrong protocol')
url = 'http://' + url # let's assume it's http, not https
log('received url %s' % url)
log('requesting headers from %s...' % url)
req = Request(url)
req.get_method = lambda: 'HEAD' # only request headers, no content
response = urlopen(req)
ctype = response.headers['content-type']
ctype_split = ctype.split('/') # split into 2 parts
log('headers received. content type is %s' % ctype)
try:
if ctype_split[0] == 'audio' or ctype_split[0] == 'video':
log('url was raw media file, playing! :)')
title = url # i guess this works? :T
play_omxplayer(url)
elif ctype_split[1] == 'x-bittorrent':
log('loading torrents not implemented.')
# this isn't implemented yet.
elif ctype_split[0] == 'text':
# here we check if it's a livestream, and if so get the RTMP url
log('checking if url is a livestream...')
live = Livestreamer()
try:
if "youtube" in url:
raise RuntimeError(
"youtube is f****d up w/ streaming, falling back to youtube-dl"
)
plugin = live.resolve_url(url)
streams = plugin.get_streams()
stream = streams.get(
"best") # fingers crossed for best quality
stream_url_types = ['rtmp', 'url'
] # things that livestreamer can have :D
for stream_type in stream_url_types:
if hasattr(stream, stream_type):
log('url is livestream!')
title = "%s (livestream)" % url
play_omxplayer(getattr(stream, stream_type))
return '', 204
except (PluginError, RuntimeError
) as e: # therefore url is not (supported) livestream
pass # continue and let youtube-dl try.
log('loading youtube-dl for further processing')
ydl = YoutubeDL({
'outtmpl': '%(id)s%(ext)s',
'restrictfilenames': True
})
ydl.add_default_info_extractors()
result = ydl.extract_info(url, download=False)
if 'entries' in result: # if video is a playlist
video = result['entries'][
0] # play the 1st video in the playlist
else:
video = result
play_omxplayer(video['url'])
title = video['title']
else:
raise DownloadError('Invalid filetype: not audio, video, or text.')
return '', 204 # success w/ no response!
except (UnicodeDecodeError, DownloadError) as e:
return _ANSI_ESCAPE_REXP.sub('', str(e)), 400 # send error message
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
info_dict = test_case.get('info_dict', {})
if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
info_dict = test_case.get('info_dict', {})
if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
print_skipping('The output file cannot be know, the "file" '
'key is missing or the info_dict is incomplete')
return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected, compat_str) and expected.startswith('md5:'):
got = 'md5:' + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(expected, got,
u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n')
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
# Check for mandatory fields that are automatically set by YoutubeDL
for key in ['webpage_url', 'extractor', 'extractor_key']:
self.assertTrue(info_dict.get(key), u'Missing field: %s' % key)
finally:
try_rm_tcs_files()
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])()
other_ies = [
get_info_extractor(ie_key)()
for ie_key in test_case.get('add_ie', [])
]
is_playlist = any(k.startswith('playlist') for k in test_case)
test_cases = test_case.get('playlist',
[] if is_playlist else [test_case])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
self.skipTest(reason)
if not ie.working():
print_skipping('IE marked as not _WORKING')
for tc in test_cases:
info_dict = tc.get('info_dict', {})
if not (info_dict.get('id') and info_dict.get('ext')):
raise Exception(
'Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file'
% (tname, ))
if 'skip' in test_case:
print_skipping(test_case['skip'])
for other_ie in other_ies:
if not other_ie.working():
print_skipping('test depends on %sIE, marked as not WORKING' %
other_ie.ie_key())
params = get_params(test_case.get('params', {}))
params['outtmpl'] = tname + '_' + params['outtmpl']
if is_playlist and 'playlist' not in test_case:
params.setdefault('extract_flat', 'in_playlist')
params.setdefault('playlistend',
test_case.get('playlist_mincount'))
params.setdefault('skip_download', True)
ydl = YoutubeDL(params, auto_init=False)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
expect_warnings(ydl, test_case.get('expected_warnings', []))
def get_tc_filename(tc):
return ydl.prepare_filename(tc.get('info_dict', {}))
res_dict = None
def try_rm_tcs_files(tcs=None):
if tcs is None:
tcs = test_cases
for tc in tcs:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
# We're not using .download here since that is just a shim
# for outside error handling, and returns the exit code
# instead of the result dict.
res_dict = ydl.extract_info(
test_case['url'],
force_generic_extractor=params.get(
'force_generic_extractor', False))
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (
compat_urllib_error.URLError, socket.timeout,
UnavailableVideoError,
compat_http_client.BadStatusLine) or (
err.exc_info[0] == compat_HTTPError
and err.exc_info[1].code == 503):
msg = getattr(err, 'msg', error_to_compat_str(err))
err.msg = '%s (%s)' % (
msg,
tname,
)
raise err
if try_num == RETRIES:
report_warning(
'%s failed due to network errors, skipping...' %
tname)
return
print(
'Retrying: {0} failed tries\n\n##########\n\n'.format(
try_num))
try_num += 1
else:
break
if is_playlist:
self.assertTrue(
res_dict['_type'] in ['playlist', 'multi_video'])
self.assertTrue('entries' in res_dict)
expect_info_dict(self, res_dict,
test_case.get('info_dict', {}))
if 'playlist_mincount' in test_case:
assertGreaterEqual(
self, len(res_dict['entries']),
test_case['playlist_mincount'],
'Expected at least %d in playlist %s, but got only %d' %
(test_case['playlist_mincount'], test_case['url'],
len(res_dict['entries'])))
if 'playlist_count' in test_case:
self.assertEqual(
len(res_dict['entries']), test_case['playlist_count'],
'Expected %d entries in playlist %s, but got %d.' % (
test_case['playlist_count'],
test_case['url'],
len(res_dict['entries']),
))
if 'playlist_duration_sum' in test_case:
got_duration = sum(e['duration'] for e in res_dict['entries'])
self.assertEqual(test_case['playlist_duration_sum'],
got_duration)
# Generalize both playlists and single videos to unified format for
# simplicity
if 'entries' not in res_dict:
res_dict['entries'] = [res_dict]
for tc_num, tc in enumerate(test_cases):
tc_res_dict = res_dict['entries'][tc_num]
# First, check test cases' data against extracted data alone
expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
# Now, check downloaded file consistency
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename),
msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
expected_minsize = tc.get('file_minsize', 10000)
if expected_minsize is not None:
if params.get('test'):
expected_minsize = max(expected_minsize, 10000)
got_fsize = os.path.getsize(tc_filename)
assertGreaterEqual(
self, got_fsize, expected_minsize,
'Expected %s to be at least %s, but it\'s only %s '
% (tc_filename, format_bytes(expected_minsize),
format_bytes(got_fsize)))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(tc['md5'], md5_for_file)
# Finally, check test cases' data again but this time against
# extracted data from info JSON file written during processing
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn),
'Missing info file %s' % info_json_fn)
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, info_dict, tc.get('info_dict', {}))
finally:
try_rm_tcs_files()
if is_playlist and res_dict is not None and res_dict.get(
'entries'):
# Remove all other files that may have been extracted if the
# extractor returns full results even with extract_flat
res_tcs = [{'info_dict': e} for e in res_dict['entries']]
try_rm_tcs_files(res_tcs)
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
is_playlist = any(k.startswith('playlist') for k in test_case)
test_cases = test_case.get(
'playlist', [] if is_playlist else [test_case])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
for tc in test_cases:
info_dict = tc.get('info_dict', {})
if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
if is_playlist and 'playlist' not in test_case:
params.setdefault('extract_flat', True)
params.setdefault('skip_download', True)
ydl = YoutubeDL(params, auto_init=False)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
expect_warnings(ydl, test_case.get('expected_warnings', []))
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
res_dict = None
def try_rm_tcs_files(tcs=None):
if tcs is None:
tcs = test_cases
for tc in tcs:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
# We're not using .download here sine that is just a shim
# for outside error handling, and returns the exit code
# instead of the result dict.
res_dict = ydl.extract_info(test_case['url'])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(u'Failed due to network errors, skipping...')
return
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
try_num += 1
else:
break
if is_playlist:
self.assertEqual(res_dict['_type'], 'playlist')
self.assertTrue('entries' in res_dict)
expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
if 'playlist_mincount' in test_case:
assertGreaterEqual(
self,
len(res_dict['entries']),
test_case['playlist_mincount'],
'Expected at least %d in playlist %s, but got only %d' % (
test_case['playlist_mincount'], test_case['url'],
len(res_dict['entries'])))
if 'playlist_count' in test_case:
self.assertEqual(
len(res_dict['entries']),
test_case['playlist_count'],
'Expected %d entries in playlist %s, but got %d.' % (
test_case['playlist_count'],
test_case['url'],
len(res_dict['entries']),
))
if 'playlist_duration_sum' in test_case:
got_duration = sum(e['duration'] for e in res_dict['entries'])
self.assertEqual(
test_case['playlist_duration_sum'], got_duration)
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
expected_minsize = tc.get('file_minsize', 10000)
if expected_minsize is not None:
if params.get('test'):
expected_minsize = max(expected_minsize, 10000)
got_fsize = os.path.getsize(tc_filename)
assertGreaterEqual(
self, got_fsize, expected_minsize,
'Expected %s to be at least %s, but it\'s only %s ' %
(tc_filename, format_bytes(expected_minsize),
format_bytes(got_fsize)))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(
os.path.exists(info_json_fn),
'Missing info file %s' % info_json_fn)
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
if is_playlist and res_dict is not None and res_dict.get('entries'):
# Remove all other files that may have been extracted if the
# extractor returns full results even with extract_flat
res_tcs = [{'info_dict': e} for e in res_dict['entries']]
try_rm_tcs_files(res_tcs)
def process_url(self, source, global_opts):
"""Main method for processing urls
This method basically hands over the configuration to YoutubeDL and repeats
the step until every source in the configuration was read
"""
ydl = YoutubeDL()
ydl.add_default_info_extractors()
sourceUrl = source['url']
sourceDescription = source.get("description", "")
self._logsource(
"Processing source: '" + sourceDescription + "' Url: '" +
sourceUrl + "'", source)
# Merge local parameters with global ones
ydl.params = copy.copy(global_opts)
ydl.params.update(source)
prefix = ""
ydl.params['match_filter'] = (None if 'match_filter' not in ydl.params
or ydl.params['match_filter'] is None
else match_filter_func(
ydl.params['match_filter']))
# Settings by commute tube over the standard settings, respect if the config sets them differently
if 'format' not in ydl.params and 'format_limit' not in ydl.params:
ydl.params[
'format'] = "bestvideo+bestaudio/best" if 'format' not in self.config else self.config[
"format"]
if 'nooverwrites' not in ydl.params:
ydl.params['nooverwrites'] = True
if 'ignoreerrors' not in ydl.params:
ydl.params['ignoreerrors'] = True
if 'download_archive' not in ydl.params:
ydl.params['download_archive'] = self.download_archive
if 'prefix' in ydl.params:
prefix = ydl.params['prefix']
ydl.params['restrictfilenames'] = True
ydl.params['logger'] = self.ydlLog
outtmpl = self.pathToDownloadFolder + "/" + prefix + \
'%(uploader)s-%(title)s.%(ext)s'
if 'outtmpl' not in ydl.params:
ydl.params['outtmpl'] = outtmpl
elif not (ydl.params['outtmpl'].startswith(self.pathToDownloadFolder)):
self._logsource(
"Prefixing custom set outtmpl with '" +
self.pathToDownloadFolder + "/" + prefix + "'", source)
ydl.params['outtmpl'] = self.pathToDownloadFolder + "/" + prefix + \
ydl.params['outtmpl']
if self.debug:
self._logsource(
"All downloads will be simulated since this is debug mode",
source)
ydl.params['simulate'] = True
ydl.download([source['url']])
"no_warnings": True,
"outtmpl": "%(title)s.%(ext)s",
"download_archive": "download_archive.txt",
"ignoreerrors": True, # ignore errors
"writethumbnail": True,
"writesubtitles": True,
"allsubtitles": True,
"min_sleep_interval": 5,
"max_sleep_interval": 15,
"addmetadata": True,
# 'listformats': True, # print a list of the formats to stdout and exit
# "forcejson": True,
}
ydl = YoutubeDL(ydl_opts)
ydl.add_default_info_extractors()
headers = {
"User-Agent":
"Mozilla/5.0 (Linux; Android 7.1.2; SM-G610M) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/71.0.3578.99 Mobile Safari/537.36"
}
PUNCT_TO_REMOVE = re.sub('[/-]', '',
string.punctuation) # Keep dashes and forward slashes
translator = str.maketrans("", "", PUNCT_TO_REMOVE)
tvdb_key = '1DE5C9B35180B706'
t = tvdb_api.Tvdb(apikey=tvdb_key)
from threading import Timer
import json
import logging
import time
import re
import random
from youtube_dl import YoutubeDL
from youtube_dl.utils import DownloadError
from minilodon.minilodon import Minilodon
bot = Minilodon("config.json")
logger = logging.getLogger(__name__)
ydl = YoutubeDL({'quiet': False, 'logger': logger, 'noplaylist': True, 'extract_flat': 'in_playlist'})
ydl.add_default_info_extractors()
spy_function = None
spy_timer = None
roll_regex = re.compile(r"([0-9]*)d([0-9]+)")
@bot.command("update", True)
def update(nick, args):
if len(args) < 4:
yield "Usage: !update <category> <key> <msg>"
return
category = args[1].lower()
key = args[2].lower()
msg = " ".join(args[3:])
if len(msg) > 254:
yield "Warning: Entry too long, message will be wrapped."
try:
from youtube_dl import YoutubeDL
from .omx_handler import OmxRequestHandler
_ytdl = YoutubeDL(dict(forceurl=True, skip_download=True, restrictfilenames=False))
_ytdl.add_default_info_extractors()
class YtDlHandler(OmxRequestHandler):
def get_location(self):
self.set_header("Access-Control-Allow-Origin", "*")
loc = self.get_argument("location")
info = _ytdl.extract_info(loc, download=False)
return info["url"]
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie._WORKING:
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case and not test_case['file']:
print_skipping('No output file specified')
return
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.fd.add_progress_hook(_hook)
test_cases = test_case.get('playlist', [test_case])
for tc in test_cases:
try_rm(tc['file'])
try_rm(tc['file'] + '.part')
try_rm(tc['file'] + '.info.json')
try:
for retry in range(1, RETRIES + 1):
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
if retry == RETRIES: raise
# Check if the exception is not a network related one
if not err.exc_info[0] in (compat_urllib_error.URLError,
socket.timeout,
UnavailableVideoError):
raise
print(
'Retrying: {0} failed tries\n\n##########\n\n'.format(
retry))
else:
break
for tc in test_cases:
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc['file']),
msg='Missing file ' + tc['file'])
self.assertTrue(tc['file'] in finished_hook_called)
self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
if 'md5' in tc:
md5_for_file = _file_md5(tc['file'])
self.assertEqual(md5_for_file, tc['md5'])
with io.open(tc['file'] + '.info.json',
encoding='utf-8') as infof:
info_dict = json.load(infof)
for (info_field, expected) in tc.get('info_dict', {}).items():
if isinstance(expected,
compat_str) and expected.startswith('md5:'):
got = 'md5:' + md5(info_dict.get(info_field))
else:
got = info_dict.get(info_field)
self.assertEqual(
expected, got,
u'invalid value for field %s, expected %r, got %r' %
(info_field, expected, got))
# If checkable fields are missing from the test case, print the info_dict
test_info_dict = dict(
(key, value if not isinstance(value, compat_str)
or len(value) < 250 else 'md5:' + md5(value))
for key, value in info_dict.items()
if value and key in ('title', 'description', 'uploader',
'upload_date', 'uploader_id',
'location'))
if not all(key in tc.get('info_dict', {}).keys()
for key in test_info_dict.keys()):
sys.stderr.write(u'\n"info_dict": ' + json.dumps(
test_info_dict, ensure_ascii=False, indent=2) + u'\n')
# Check for the presence of mandatory fields
for key in ('id', 'url', 'title', 'ext'):
self.assertTrue(key in info_dict.keys() and info_dict[key])
finally:
for tc in test_cases:
try_rm(tc['file'])
try_rm(tc['file'] + '.part')
try_rm(tc['file'] + '.info.json')
def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [
get_info_extractor(ie_key)
for ie_key in test_case.get('add_ie', [])
]
def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working():
print_skipping('IE marked as not _WORKING')
return
if 'playlist' not in test_case:
info_dict = test_case.get('info_dict', {})
if not test_case.get('file') and not (info_dict.get('id')
and info_dict.get('ext')):
raise Exception(
'Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?'
)
if 'skip' in test_case:
print_skipping(test_case['skip'])
return
for other_ie in other_ies:
if not other_ie.working():
print_skipping(u'test depends on %sIE, marked as not WORKING' %
other_ie.ie_key())
return
params = get_params(test_case.get('params', {}))
ydl = YoutubeDL(params)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
ydl.add_progress_hook(_hook)
def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(
tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files():
for tc in test_cases:
tc_filename = get_tc_filename(tc)
try_rm(tc_filename)
try_rm(tc_filename + '.part')
try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
try_rm_tcs_files()
try:
try_num = 1
while True:
try:
ydl.download([test_case['url']])
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
if not err.exc_info[0] in (
compat_urllib_error.URLError, socket.timeout,
UnavailableVideoError,
compat_http_client.BadStatusLine) or (
err.exc_info[0] == compat_HTTPError
and err.exc_info[1].code == 503):
raise
if try_num == RETRIES:
report_warning(
u'Failed due to network errors, skipping...')
return
print(
'Retrying: {0} failed tries\n\n##########\n\n'.format(
try_num))
try_num += 1
else:
break
for tc in test_cases:
tc_filename = get_tc_filename(tc)
if not test_case.get('params', {}).get('skip_download', False):
self.assertTrue(os.path.exists(tc_filename),
msg='Missing file ' + tc_filename)
self.assertTrue(tc_filename in finished_hook_called)
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
self.assertTrue(os.path.exists(info_json_fn))
if 'md5' in tc:
md5_for_file = _file_md5(tc_filename)
self.assertEqual(md5_for_file, tc['md5'])
with io.open(info_json_fn, encoding='utf-8') as infof:
info_dict = json.load(infof)
expect_info_dict(self, tc.get('info_dict', {}), info_dict)
finally:
try_rm_tcs_files()
