def record_stream(channel, output_dir): channels = get_channels("world", 2) channel_url = "http://www.youtube.com/watch?v={}".format(channels[channel - 1]) print("Downloading from", channel_url) ydl = YoutubeDL() ydl.add_default_info_extractors() info = ydl.extract_info(channel_url, download=False) if info["uploader_id"] != "coachella": print("Uploader is not coachella, stopping") exit() filename = "channel{}-{}.ts".format(channel, int(time.time())) streamlink = Streamlink() streamlink.set_loglevel("debug") streamlink.set_logoutput(output_dir + "/" + filename + ".log") streamlink.set_option("hls-live-edge", 9999999) streamlink.set_option("hls-segment-attempts", 99) streamlink.set_option("hls-segment-threads", 5) streamlink.set_option("hls-segment-timeout", 9999) streams = streamlink.streams(channel_url) stream = streams["best"] print("Found stream {}".format(stream.url)) print("Writing stream to {}".format(output_dir + "/" + filename)) with open(output_dir + "/" + filename, 'wb') as out_file, stream.open() as in_stream: while True: data = in_stream.read(1024) if not data: return out_file.write(data)
def get_video_url(params): """Get video URL and start video player""" url_selected = '' all_datas_videos_quality = [] all_datas_videos_path = [] videos_html = utils.get_webcontent(params.video_url) videos_soup = bs(videos_html, 'html.parser') list_videos = videos_soup.find( 'ul', class_='nav nav-tabs').find_all('a') for video in list_videos: if '#video-' in video.get('href'): # Find a better solution to strip all_datas_videos_quality.append(video.get_text().strip()) # Get link value_jwplayer_id = video.get('data-jwplayer-id') # Case mp4 if value_jwplayer_id != '': list_streams = videos_soup.find_all( 'div', class_='jwplayer') for stream in list_streams: if stream.get('id') == value_jwplayer_id: url = stream.get('data-source') # Cas Yt else: video_id = re.compile( 'youtube.com/embed/(.*?)\?').findall(videos_html)[0] url = resolver.get_stream_youtube(video_id, False) all_datas_videos_path.append(url) # Get link from FranceTV elif '#ftv-player-' in video.get('href'): # Find a better solution to strip all_datas_videos_quality.append(video.get_text().strip()) # Get link value_ftvlayer_id = video.get('data-ftvplayer-id') list_streams = videos_soup.find_all( 'iframe', class_='embed-responsive-item') for stream in list_streams: if stream.get('id') == value_ftvlayer_id: url_id = stream.get('src') ydl = YoutubeDL() ydl.add_default_info_extractors() with ydl: result = ydl.extract_info( url_id, download=False) for format_video in result['formats']: url = format_video['url'] all_datas_videos_path.append(url) if len(all_datas_videos_quality) > 1: seleted_item = common.sp.xbmcgui.Dialog().select( common.GETTEXT('Choose video quality'), all_datas_videos_quality) if seleted_item == -1: return '' url_selected = all_datas_videos_path[seleted_item] return url_selected else: return all_datas_videos_path[0]
class YoutubeDLWrapper(object): """ Used to wrap youtubedl import, since youtubedl currently overrides global HTMLParser.locatestarttagend regex with a different regex that doesn't quite work. This wrapper ensures that this regex is only set for YoutubeDL and unset otherwise """ def __init__(self): import HTMLParser as htmlparser self.htmlparser = htmlparser self.orig_tagregex = htmlparser.locatestarttagend from youtube_dl import YoutubeDL as YoutubeDL self.ydl_tagregex = htmlparser.locatestarttagend htmlparser.locatestarttagend = self.orig_tagregex self.ydl = YoutubeDL(dict(simulate=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def extract_info(self, url): info = None try: self.htmlparser.locatestarttagend = self.ydl_tagregex info = self.ydl.extract_info(url) finally: self.htmlparser.locatestarttagend = self.orig_tagregex return info
def YouPlay(vidUrl,isPlaylist=False): VIDEO_URL = vidUrl ydl_opt={"extract_flat":True,'quiet':True} ydl = YoutubeDL(ydl_opt) ydl.add_default_info_extractors() if(not isPlaylist): info = ydl.extract_info(VIDEO_URL,download=False) title=info['title'] print(title) url = info['formats'][0]['url'] instance = vlc.Instance('--no-video --quiet') Vars.player = instance.media_player_new() media = instance.media_new(url) Vars.player.set_media(media) Vars.player.play() else: ydl_opt={"extract_flat":True,'quiet':True,'playlistrandom':True} ydl = YoutubeDL(ydl_opt) info = ydl.extract_info(VIDEO_URL,download=False) entries=info['entries'] Vars.playlistPlayer=PlaylistPlaying(Vars.player, entries) Vars.playlistPlayer.start()
class ScramDLThread(threading.Thread): def __init__(self, urls, addendum_params=None): threading.Thread.__init__(self) override_params = dict() override_params['writeinfojson'] = False override_params['test'] = False override_params['format'] = 'mp4' if addendum_params is not None: override_params.update(addendum_params) params = get_params(override_params) self.scram_dl = YoutubeDL(params) self.scram_dl.add_progress_hook(self.hook_progress) self.scram_dl.add_default_info_extractors() self.is_finished = False self.urls = urls def hook_progress(self, status): if status['status'] == 'finished': self.is_finished = True else: self.is_finished = False def run(self): self.scram_dl.download(self.urls)
def download(url, destination, logger): opts = { 'format': 'm4a/bestaudio', 'outtmpl': destination + '/%(title)s.%(ext)s', 'logger': logger, 'progress_hooks': [dlProgress], 'postprocessors': [{ 'key': 'MetadataFromTitle', 'titleformat': '%(artist)s - %(title)s' }, { 'key': 'FFmpegMetadata' }], 'ffmpeg_location': resource_path('ffmpeg.exe') } ydl = YoutubeDL(opts) ydl.add_default_info_extractors() try: ydl.download([url]) except: app.enableButton("Download") app.enableEntry("URL") app.enableDirectoryEntry("destination") app.queueFunction(writeOutput, sys.exc_info())
def process_item(self, item, spider): fb_video_url = item.get('fb_video_url') if fb_video_url: ydl = YoutubeDL() ydl.add_default_info_extractors() ydl.extract_info(fb_video_url) return item
class DownloadProcess(multiprocessing.Process): """ Actual download process which calls youtube-dl. You should never need to interact with this directly. """ def __init__(self, url, status_queue, output_writer): super(DownloadProcess, self).__init__() self.url = url self.status_queue = status_queue self.info = {} sys.stdout = output_writer sys.stderr = output_writer self.downloader = YoutubeDL({"progress_hooks": [self.update_status]}) self.downloader.add_default_info_extractors() def update_status(self, status): self.status_queue.put(status) def run(self): self.info = self.downloader.extract_info(self.url) def stop(self): # Kill any child processes that may have spawned (e.g. ffmpeg) import psutil, signal s = psutil.Process() for child in s.children(recursive=True): child.send_signal(signal.SIGINT)
def run(event, context): url = event['url'] ydl_opts = { 'outtmpl': '/tmp/%(title)s-%(id)s.%(ext)s', 'format': 'bestaudio/best', 'postprocessors': [{ 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', }], 'logger': Logger(), 'progress_hooks': [processing_hook], 'ffmpeg_location': 'ffmpeg/', } ydl = YoutubeDL(ydl_opts) ydl.add_default_info_extractors() info = ydl.extract_info(url, True) response = upload(info) return {"url": response}
class PyJizzParser(object): def __init__(self, model): self.model = model self.model.parser = self self.ydl = YoutubeDL() self.ydl.add_default_info_extractors() def parseCategories(self): c = PornHubCategoryParser(self.model) c.run() def parseCategoryPage(self, category, page = 1): if page == 0 or page == 1: url = self.model.categories_url[category] else: url = "{site}{page_url}{page}".format( site = self.model.site, page_url = self.model.p**n[category]['page_url'], page = page) print("page parser creating for page", page) p = PornHubPageParser(self.model, url, category, page) p.run() print("page parser exit for page", page) def getInfo(self, vkey): info = self.ydl.extract_info('http://www.pornhub.com/view_video.php?viewkey={v}'.format(v = vkey), download=False) return info
class YoutubeDLWrapper(object): """ Used to wrap youtubedl import, since youtubedl currently overrides global HTMLParser.locatestarttagend regex with a different regex that doesn't quite work. This wrapper ensures that this regex is only set for YoutubeDL and unset otherwise """ def __init__(self): import HTMLParser as htmlparser self.htmlparser = htmlparser self.orig_tagregex = htmlparser.locatestarttagend from youtube_dl import YoutubeDL as YoutubeDL self.ydl_tagregex = htmlparser.locatestarttagend htmlparser.locatestarttagend = self.orig_tagregex self.ydl = YoutubeDL( dict(simulate=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def extract_info(self, url): info = None try: self.htmlparser.locatestarttagend = self.ydl_tagregex info = self.ydl.extract_info(url) finally: self.htmlparser.locatestarttagend = self.orig_tagregex return info
def process_url(self, source, global_opts): """Main method for processing urls This method basically hands over the configuration to YoutubeDL and repeats the step until every source in the configuration was read """ ydl = YoutubeDL() ydl.add_default_info_extractors() sourceUrl = source['url'] sourceDescription = source.get("description", "") self._logsource( "Processing source: '" + sourceDescription + "' Url: '" + sourceUrl + "'", source) # Merge local parameters with global ones ydl.params = copy.copy(global_opts) ydl.params.update(source) prefix = "" ydl.params['match_filter'] = ( None if 'match_filter' not in ydl.params or ydl.params['match_filter'] is None else match_filter_func(ydl.params['match_filter'])) # Settings by commute tube over the standard settings, respect if the config sets them differently if 'format' not in ydl.params and 'format_limit' not in ydl.params: ydl.params['format'] = "bestvideo+bestaudio/best" if 'format' not in self.config else self.config["format"] if 'nooverwrites' not in ydl.params: ydl.params['nooverwrites'] = True if 'ignoreerrors' not in ydl.params: ydl.params['ignoreerrors'] = True if 'download_archive' not in ydl.params: ydl.params['download_archive'] = self.download_archive if 'prefix' in ydl.params: prefix = ydl.params['prefix'] ydl.params['restrictfilenames'] = True ydl.params['logger'] = self.ydlLog outtmpl = self.pathToDownloadFolder + "/" + prefix + \ '%(uploader)s-%(title)s.%(ext)s' if 'outtmpl' not in ydl.params: ydl.params['outtmpl'] = outtmpl elif not (ydl.params['outtmpl'].startswith(self.pathToDownloadFolder)): self._logsource("Prefixing custom set outtmpl with '" + self.pathToDownloadFolder + "/" + prefix + "'", source) ydl.params['outtmpl'] = self.pathToDownloadFolder + "/" + prefix + \ ydl.params['outtmpl'] if self.debug: self._logsource( "All downloads will be simulated since this is debug mode", source) ydl.params['simulate'] = True ydl.download([source['url']])
def sys_main(): ydl = YoutubeDL() ydl.add_default_info_extractors() info = ydl.extract_info('https://www.youtube.com/watch?v=-tzvebu6U08', download=False) for x in info['formats']: print(x['format'], x['url']) return 0
def get_info(url): ydl = YoutubeDL({ 'forceurl': True, 'quiet': True }) ydl.add_default_info_extractors() return ydl.extract_info(url, download=False)
def get_youtube_info(url): ydl = YoutubeDL() ydl.add_default_info_extractors() try: info = ydl.extract_info(url, download=False) except: return None return info
def get_youtube_info(url): ydl = YoutubeDL() ydl.add_default_info_extractors() try: info = ydl.extract_info(url, download=False) except: return None return info
def youtube_downloader(url): ydl_options = {'outtmpl': video_files_path + '%(title)s.%(ext)s'} ydl = YoutubeDL(ydl_options) ydl.add_default_info_extractors() info = ydl.extract_info( url=url, download=True, ) return ydl.prepare_filename(info)
def get_video_url(params): """Get video URL and start video player""" url_selected = '' all_datas_videos_quality = [] all_datas_videos_path = [] videos_html = utils.get_webcontent(params.video_url) videos_soup = bs(videos_html, 'html.parser') list_videos = videos_soup.find('ul', class_='nav nav-tabs').find_all('a') for video in list_videos: if '#video-' in video.get('href'): # Find a better solution to strip all_datas_videos_quality.append(video.get_text().strip()) # Get link value_jwplayer_id = video.get('data-jwplayer-id') # Case mp4 if value_jwplayer_id != '': list_streams = videos_soup.find_all('div', class_='jwplayer') for stream in list_streams: if stream.get('id') == value_jwplayer_id: url = stream.get('data-source') # Cas Yt else: video_id = re.compile('youtube.com/embed/(.*?)\?').findall( videos_html)[0] url = resolver.get_stream_youtube(video_id, False) all_datas_videos_path.append(url) # Get link from FranceTV elif '#ftv-player-' in video.get('href'): # Find a better solution to strip all_datas_videos_quality.append(video.get_text().strip()) # Get link value_ftvlayer_id = video.get('data-ftvplayer-id') list_streams = videos_soup.find_all('iframe', class_='embed-responsive-item') for stream in list_streams: if stream.get('id') == value_ftvlayer_id: url_id = stream.get('src') ydl = YoutubeDL() ydl.add_default_info_extractors() with ydl: result = ydl.extract_info(url_id, download=False) for format_video in result['formats']: url = format_video['url'] all_datas_videos_path.append(url) if len(all_datas_videos_quality) > 1: seleted_item = common.sp.xbmcgui.Dialog().select( common.GETTEXT('Choose video quality'), all_datas_videos_quality) if seleted_item == -1: return '' url_selected = all_datas_videos_path[seleted_item] return url_selected else: return all_datas_videos_path[0]
def run(argument): if argument['type'] == 'magnet': openWith = utils.getSetting('openMagnetWith') logger.info('Sharing magnet with %s' % openWith) if openWith == 'Elementum': utils.callPlugin('plugin://plugin.video.elementum/playuri?uri=' + argument['data'] + ')') elif openWith == 'Torrenter V2': utils.callPlugin( 'plugin://plugin.video.torrenter/?action=playSTRM&url=' + argument['data'] + ')') elif openWith == 'Quasar': utils.callPlugin('plugin://plugin.video.quasar/playuri?uri=' + argument['data'] + ')') elif argument['type'] == 'unresolvedurl': url = urllib.unquote(argument['data']) logger.info('Trying to resolve with urlresolver: %s' % url) stream_url = urlresolver.HostedMediaFile(url=url).resolve() if not stream_url: if have_youtube_dl == "false": logger.error("Url not resolved and no YoutubeDL: %s" % url) else: logger.info('Trying to resolve with YoutubeDL: %s' % url) YoutubeDL_resolver = YoutubeDL({'format': 'best'}) YoutubeDL_resolver.add_default_info_extractors() result = YoutubeDL_resolver.extract_info(url, download=False) if "url" in result: stream_url = result["url"] logger.info("Url resolved by YoutubeDL: %s" % result) else: logger.error("Url not resolved: %s" % url) else: logger.info("Url resolved by urlResolver: %s" % stream_url) if stream_url: if (not 'queue' in argument) or (argument['queue'] == "false") or ( not xbmc.Player().isPlaying()): logger.info('Playing resolved url: %s' % stream_url) xbmc.Player().play(stream_url) else: if xbmc.Player().isPlayingAudio(): logger.info('Queuing to music resolved url: %s' % stream_url) xbmc.PlayList(xbmc.PLAYLIST_MUSIC).add(stream_url) else: logger.info('Queuing to video resolved url: %s' % stream_url) xbmc.PlayList(xbmc.PLAYLIST_VIDEO).add(stream_url) else: dialog = xbmcgui.Dialog() dialog.notification(utils.ADDON_NAME, utils.translation(32006), xbmcgui.NOTIFICATION_INFO, 5000)
def extractMeta(link_out): global gen_info gen_info = [] ydl = YoutubeDL() ydl.add_default_info_extractors() for link in link_out: meta = ydl.extract_info(link, download=False) for formats in meta['formats']: if formats['format_id'] == '18': gen_info.append([ meta['id'], meta['title'], meta['thumbnail'], formats['url'] ])
def get_stream_youtube(video_id, isDownloadVideo): url_youtube = URL_YOUTUBE % video_id if isDownloadVideo == True: return url_youtube ydl = YoutubeDL() ydl.add_default_info_extractors() with ydl: result = ydl.extract_info(url_youtube, download=False) for format_video in result['formats']: url_live = format_video['url'] return url_live
class VideoLoader(BaseLoader): CONTENT_TYPE = 'application/vnd.youtube-dl_formats+json' def __init__(self): try: from youtube_dl import YoutubeDL as YoutubeDL except ImportError: self.ydl = None return self.ydl = YoutubeDL( dict(simulate=True, quiet=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def load_resource(self, cdx, params): load_url = cdx.get('load_url') if not load_url: return None if params.get('content_type') != self.CONTENT_TYPE: return None if not self.ydl: return None info = self.ydl.extract_info(load_url) info_buff = json.dumps(info) info_buff = info_buff.encode('utf-8') warc_headers = {} schema, rest = load_url.split('://', 1) target_url = 'metadata://' + rest dt = timestamp_to_datetime(cdx['timestamp']) warc_headers['WARC-Type'] = 'metadata' warc_headers['WARC-Record-ID'] = self._make_warc_id() warc_headers['WARC-Target-URI'] = target_url warc_headers['WARC-Date'] = datetime_to_iso_date(dt) warc_headers['Content-Type'] = self.CONTENT_TYPE warc_headers['Content-Length'] = str(len(info_buff)) warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items()) return warc_headers, None, BytesIO(info_buff)
def download(link, files_present, avg): '''gets the playlist and Downloads the videos that i dont have''' #url = 'https://www.youtube.com/watch?v=MCs5OvhV9S4' url = 'https://www.youtube.com/playlist?list=PLwyG5wA5gIzjhW36BxGBoQwUZHnPDFux3' ydl=YoutubeDL() ydl.add_default_info_extractors() playlist = ydl.extract_info(url, download=False) for video in playlist['entries']: import ipdb; ipdb.set_trace() if video['title'] in files_present: print ("Video #{} {} is present and will be ignored").format(video['playlist_index'], video['title']) else: print ("currently downloading: {}").format(video['title']) ydl.download(video['webpage_url'])
def download_youtube(url,filepath,params=None): tmp_filepath = compat_str(filepath) print "download to",tmp_filepath params = params or settings.youtube_params params.update({"outtmpl":tmp_filepath,"daterange":DateRange(None,None)}) y = YoutubeDL(params) #y = YoutubeDL({"format":"18/34/35/5/17","outtmpl":filepath}) #y.print_debug_header() y.add_default_info_extractors() y.add_post_processor(FFmpegExtractAudioPP(preferredcodec="m4a",preferredquality=5, nopostoverwrites=False)) value = y.download([url]) #cmd = 'youtube-dl {url} --extract-audio --audio-format wav -o {filepath}'.format(url=url,filepath=filepath) #print cmd #result = subprocess.call(cmd,shell=True) #print result return True
class VideoLoader(BaseLoader): CONTENT_TYPE = 'application/vnd.youtube-dl_formats+json' def __init__(self): try: from youtube_dl import YoutubeDL as YoutubeDL except ImportError: self.ydl = None return self.ydl = YoutubeDL(dict(simulate=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def load_resource(self, cdx, params): load_url = cdx.get('load_url') if not load_url: return None if params.get('content_type') != self.CONTENT_TYPE: return None if not self.ydl: return None info = self.ydl.extract_info(load_url) info_buff = json.dumps(info) info_buff = info_buff.encode('utf-8') warc_headers = {} schema, rest = load_url.split('://', 1) target_url = 'metadata://' + rest dt = timestamp_to_datetime(cdx['timestamp']) warc_headers['WARC-Type'] = 'metadata' warc_headers['WARC-Record-ID'] = self._make_warc_id() warc_headers['WARC-Target-URI'] = target_url warc_headers['WARC-Date'] = datetime_to_iso_date(dt) warc_headers['Content-Type'] = self.CONTENT_TYPE warc_headers['Content-Length'] = str(len(info_buff)) warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items()) return warc_headers, None, BytesIO(info_buff)
def _download_restricted(url, filename, age): """ Returns true iff the file has been downloaded """ params = { 'age_limit': age, 'skip_download': True, 'writeinfojson': True, "outtmpl": "%(id)s.%(ext)s", } ydl = YoutubeDL(params) ydl.add_default_info_extractors() json_filename = filename + '.info.json' try_rm(json_filename) ydl.download([url]) res = os.path.exists(json_filename) try_rm(json_filename) return res
def on_task_output(self, task, config): import youtube_dl.YoutubeDL from youtube_dl.utils import ExtractorError class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] super(YoutubeDL, self).__init__(*args, **kwargs) def report_warning(self, message): raise ExtractorError(message) def process_info(self, info_dict): self.processed_info_dicts.append(info_dict) return super(YoutubeDL, self).process_info(info_dict) for entry in task.accepted: if task.options.test: log.info('Would download %s' % entry['title']) else: try: outtmpl = entry.render(config['path']) + '/' + pathscrub(entry.render(config['template']) + '.%(ext)s', filename=True) log.info("Output file: %s" % outtmpl) except RenderError as e: log.error('Error setting output file: %s' % e) entry.fail('Error setting output file: %s' % e) params = {'quiet': True, 'outtmpl': outtmpl} if 'username' in config and 'password' in config: params.update({'username': config['username'], 'password': config['password']}) elif 'username' in config or 'password' in config: log.error('Both username and password is required') if 'videopassword' in config: params.update({'videopassword': config['videopassword']}) if 'title' in config: params.update({'title': config['title']}) ydl = YoutubeDL(params) ydl.add_default_info_extractors() log.info('Downloading %s' % entry['title']) try: ydl.download([entry['url']]) except ExtractorError as e: log.error('Youtube-DL was unable to download the video. Error message %s' % e.message) entry.fail('Youtube-DL was unable to download the video. Error message %s' % e.message) except Exception as e: log.error('Youtube-DL failed. Error message %s' % e.message) entry.fail('Youtube-DL failed. Error message %s' % e.message)
def _download_restricted(url, filename, age): """ Returns true iff the file has been downloaded """ params = { 'age_limit': age, 'skip_download': True, 'writeinfojson': True, "outtmpl": "%(id)s.%(ext)s", } ydl = YoutubeDL(params) ydl.add_default_info_extractors() json_filename = filename + '.info.json' try_rm(json_filename) ydl.download([url]) res = os.path.exists(json_filename) try_rm(json_filename) return res
def __init__(self, dl_id, url): self.dl_id = dl_id self.url = url self.start_time = datetime.datetime.utcnow() self.download_proc = None self.status_queue = multiprocessing.Queue() self.status = {} self.child_output = OutputWriter() self.log = "" info_downloader = YoutubeDL({"quiet": True}) info_downloader.add_default_info_extractors() self.info = info_downloader.extract_info(self.url, download=False)
def processUrl(self, source): ydl = YoutubeDL() ydl.add_default_info_extractors() sourceUrl = source['url'].decode() sourceDescription = "" if 'description' in source: sourceDescription = source['description'].decode() self.log.info( "Processing source: '" + sourceDescription + "' Url: '" + sourceUrl + "'") ydl.params = source prefix = "" if 'format' not in ydl.params and 'format_limit' not in ydl.params: ydl.params['format'] = "bestvideo+bestaudio" if 'nooverwrites' not in ydl.params: ydl.params['nooverwrites'] = True if 'ignoreerrors' not in ydl.params: ydl.params['ignoreerrors'] = True if 'download_archive' not in ydl.params: ydl.params['download_archive'] = "already_downloaded.txt" if 'prefix' in ydl.params: prefix = ydl.params['prefix'] ydl.params['restrictfilenames'] = True ydl.params['logger'] = self.ydlLog outtmpl = self.pathToDownloadFolder + "/" + prefix + \ u'%(uploader)s-%(title)s-%(id)s.%(ext)s' if 'outtmpl' not in ydl.params: ydl.params['outtmpl'] = outtmpl if self.debug is True: self.log.debug( "All downloads will be simulated since this is debug mode") ydl.params['simulate'] = True ydl.download([source['url']])
def get_videos_from_playlist(self): playlist = self.get_YT_playlists() request = self.youtube_client.playlistItems().list( part="snippet", playlistId=self.playlists[playlist], maxResults=50) response = request.execute() ydl = YoutubeDL() ydl.add_default_info_extractors() for item in response["items"]: info = ydl.extract_info('http://www.youtube.com/watch?v=' + item["snippet"]["resourceId"]["videoId"], download=False) uri = self.get_Spotify_song_uri(info["track"], info["artist"]) if uri is None: continue self.uris.append(uri)
def resolve_with_youtube_dl(url, parameters, action): youtube_dl_resolver = YoutubeDL(parameters) youtube_dl_resolver.add_default_info_extractors() try: result = youtube_dl_resolver.extract_info(url, download=False) if result is None: result = {} except Exception as e: logger.error(u'Error with YoutubeDL: %s' % e) result = {} logger.info(u'YoutubeDL full result: %s' % result) if 'entries' in result: logger.info(u'Playlist resolved by YoutubeDL: %s items' % len(result['entries'])) item_list = [] for entry in result['entries']: if entry is not None and 'url' in entry: item_list.append(entry) logger.info(u'Media found: %s' % entry['url']) if len(item_list) > 0: utils.play_items(item_list, action) return True else: logger.info(u'No playable urls in the playlist') if 'url' in result: logger.info(u'Url resolved by YoutubeDL: %s' % result['url']) utils.play_url(result['url'], action, result) return True if 'requested_formats' in result: if have_adaptive_plugin: logger.info(u'Adaptive plugin enabled looking for dash content') for entry in result['requested_formats']: if 'container' in entry and 'manifest_url' in entry: if 'dash' in entry['container']: logger.info(u'Url resolved by YoutubeDL: %s' % entry['manifest_url']) utils.play_url(entry['manifest_url'], action, result, True) return True for entry in result['requested_formats']: if 'protocol' in entry and 'manifest_url' in entry: if 'm3u8' in entry['protocol']: logger.info(u'Url resolved by YoutubeDL: %s' % entry['manifest_url']) utils.play_url(entry['manifest_url'], action, result) return True return False
def processUrl(self, source): ydl = YoutubeDL() ydl.add_default_info_extractors() sourceUrl = source['url'].decode() sourceDescription = "" if 'description' in source: sourceDescription = source['description'].decode() self.log.info("Processing source: '" + sourceDescription + "' Url: '" + sourceUrl + "'") ydl.params = source prefix = "" if 'format' not in ydl.params and 'format_limit' not in ydl.params: ydl.params['format'] = "bestvideo+bestaudio" if 'nooverwrites' not in ydl.params: ydl.params['nooverwrites'] = True if 'ignoreerrors' not in ydl.params: ydl.params['ignoreerrors'] = True if 'download_archive' not in ydl.params: ydl.params['download_archive'] = "already_downloaded.txt" if 'prefix' in ydl.params: prefix = ydl.params['prefix'] ydl.params['restrictfilenames'] = True ydl.params['logger'] = self.ydlLog outtmpl = self.pathToDownloadFolder + "/" + prefix + \ u'%(uploader)s-%(title)s-%(id)s.%(ext)s' if 'outtmpl' not in ydl.params: ydl.params['outtmpl'] = outtmpl if self.debug is True: self.log.debug( "All downloads will be simulated since this is debug mode") ydl.params['simulate'] = True ydl.download([source['url']])
class YoutubeDLWrapper(object): #pragma: no cover """ YoutubeDL wrapper, inits youtubee-dl if it is available """ def __init__(self): try: from youtube_dl import YoutubeDL as YoutubeDL except ImportError: self.ydl = None return self.ydl = YoutubeDL(dict(simulate=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def extract_info(self, url): if not self.ydl: return None info = self.ydl.extract_info(url) return info
class YoutubeDLWrapper(object): #pragma: no cover """ YoutubeDL wrapper, inits youtubee-dl if it is available """ def __init__(self): try: from youtube_dl import YoutubeDL as YoutubeDL except ImportError: self.ydl = None return self.ydl = YoutubeDL( dict(simulate=True, youtube_include_dash_manifest=False)) self.ydl.add_default_info_extractors() def extract_info(self, url): if not self.ydl: return None info = self.ydl.extract_info(url) return info
def indir(self): os.getcwd() os.chdir( os.getcwd() ) # Videoların programın olduğu dizine kaydetmesini bu iki satır sağlıyor. if self.veri.text().startswith("https://www.youtube.com/"): ydl_ayarlar = {'noplaylist': True, 'progress_hooks': [self.hook]} QMessageBox.information( self, "Videonuz işlenmeye hazır", "İşleme süreci indirdiğiniz videonun boyutuna bağlıdır bekleme süreniz de buna göre " "değişebilir.İşleme süreci bittiğinde bar dolmaya başlayacak ve video indirilecektir." ) self.setWindowTitle("Videonuz İşleniyor....") ydl = YoutubeDL(ydl_ayarlar) ydl.add_default_info_extractors() info = ydl.extract_info(self.veri.text(), download=True) self.setWindowTitle("Videonuz İndiriliyor....") self.buton.setEnabled(False) self.bariSekillendir() indirilen_oge = info["title"] self.liste.insertItem(0, indirilen_oge) self.setWindowTitle("Youtube [HD] MP4 İndirici | Cankat Özçakır") QMessageBox.information( self, "İndirme Başarılı!", "İndirme işlemini başarıyla gerçekleştirdiniz.") self.buton.setEnabled(True) elif not self.veri.text(): self.bilgilendirme.setText("Herhangi bir adres belirtmediniz.") QTimer.singleShot(2000, self.bilgilendirmeYazisi) else: self.bilgilendirme.setText( "Yalnızca Youtube'dan video indirebilirsiniz.") QTimer.singleShot(2000, self.bilgilendirmeYazisi)
def download_video(channel: dict, video: dict) -> pathlib.Path: """ Download a video (and associated posters/etc) to it's channel's directory. :param channel: :param video: A YoutubeDL info entry dictionary :return: """ # YoutubeDL expects specific options, add onto the default options config = get_downloader_config() options = dict(config) directory = get_absolute_media_path(channel['directory']) options['outtmpl'] = f'{directory}/{config["file_name_format"]}' ydl = YoutubeDL(options) ydl.add_default_info_extractors() source_id = video['id'] url = f'https://www.youtube.com/watch?v={source_id}' entry = ydl.extract_info(url, download=True, process=True) final_filename = ydl.prepare_filename(entry) final_filename = pathlib.Path(final_filename) return final_filename
def _parse_url(url, callback): download_tag = DownloadTag(url) params = {"encoding": "utf8", "format": "best", "quiet": True, } ydl = YoutubeDL(params) ydl.add_default_info_extractors() try: info_dict = ydl.extract_info(url, download=False) except DownloadError as e: _g_logger.exception(e) download_tag.error = "Download Error. See error.txt for details" else: download_tag.ext = "." + info_dict["ext"] download_tag.title = info_dict["title"] download_tag.thumbnail_url = info_dict["thumbnail"] download_tag.video_url = info_dict["url"] wx.CallAfter(callback, download_tag)
def lambda_handler(event, context): url = event['url'] ### Need to be changed for Api Gateway POST request ydl_opts = { 'outtmpl': '/tmp/%(title)s-%(id)s.%(ext)s', 'format': 'bestaudio/best', 'postprocessors': [{ 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', }], 'logger': MyLogger(), 'progress_hooks': [my_hook], 'ffmpeg_location': 'ffmpeg/', } ydl = YoutubeDL(ydl_opts) ydl.add_default_info_extractors() info = ydl.extract_info(url, True) response = upload(info) return {"url": response}
def download(download): """Downloads the youtube video from the url Args: download: A DownloadRequest. Returns: A (file name, video title) tuple. The file name is ONLY the file name, and does not include the file path. """ downloader = YoutubeDL() downloader.add_default_info_extractors() downloader.params['outtmpl'] = os.path.join(os.getcwd(), 'temp/%(id)s.%(ext)s') downloader.params['verbose'] = True downloader.params['cachedir'] = None downloader.params['noplaylist'] = True downloader.params['max_downloads'] = 1 if download.is_audio_only(): downloader.params['format'] = 'bestaudio' else: # We are only going to support downloading .mp4 videos. downloader.params['format'] = 'mp4' if download.get_force_mp4_filetype(): downloader.params['format'] = 'mp4' info = downloader.extract_info(download.get_url()) file_name = downloader.prepare_filename(info) file_name = file_name.encode('ascii', 'ignore') title = info.get('title', os.path.basename(file_name)) return file_name, title
class WhydScraper(): """ All your tracks are belong to us. """ def __init__(self): self.ydl = YoutubeDL(YDL_PARAMS) self.ydl.add_default_info_extractors() def _make_soup(self, url): """ Helper to make a BeautifulSoup object. :param url: URL to scrap. """ try: return BeautifulSoup(requests.get(url).text) except ValueError: logging.warning('The URL you provided is not valid : "%s"' % url) return None def scrap_playlists(self, username): """ Scrap user playlists from his profile page. :param username: User name (slug) OR id (/u/xxxx). """ soup = self._make_soup('{root_url}/{username}/playlists'.format( root_url=ROOT_URL, username=username)) pl_tags = soup.find_all('li', class_='playlist') return [li.a['href'].split('/')[-1] for li in pl_tags] def get_playlist_by_id(self, username, pl_id, limit=PLAYLIST_LIMIT): """ Get tracklist by Username + ID.""" res = requests.get( '{root_url}/{username}/playlist/{pl_id}?format=json&limit={limit}'.format( root_url=ROOT_URL, username=username, pl_id=pl_id, limit=limit)) tracks = self._format_json(res.json()) return tracks def get_playlist_by_url(self, pl_url, limit=PLAYLIST_LIMIT): """ Get tracklist by URL. """ res = requests.get('{url}?format=json&limit={limit}'.format( url=pl_url, limit=limit)) tracks = self._format_json(res.json()) return tracks def _format_json(self, json): """ Parse JSON and return formated dict. """ # format urls for track in json: for e in PROVIDERS: if PROVIDERS[e]['prefix'] in track['eId']: track['eId'] = track['eId'].replace( PROVIDERS[e]['prefix'], PROVIDERS[e]['root_url']).split('#')[0] try: playlist = track['pl']['name'] except KeyError: playlist = 'None' return [{'playlist': playlist, 'name': track['name'], 'url': track['eId']} for track in json] def get_user_stream(self, username, limit='20'): """ Get user tracklist from its added tracks stream. :param username: User nickname (slug) OR id (/u/xxxx). :param limit: Maximum number of tracks to retrieve. """ res = requests.get( '{root_url}/{username}?format=json&limit={limit}'.format( root_url=ROOT_URL, username=username, limit=limit)) tracks = self._format_json(res.json()) return tracks def get_user_likes(self, username): """ Get user tracklist from its liked tracks stream. :param username: User nickname (slug) OR id (/u/xxxx). """ res = requests.get( '{root_url}/{username}/likes?format=json'.format( root_url=ROOT_URL, username=username)) tracks = self._format_json(res.json()) return tracks def download(self, url, outdir, dry_run=False): """ Download the audio version of a given track url. :param url: Track URL. :param outdir: Output directory. """ if dry_run: print(url) return # output root folder try: os.mkdir('output') except OSError: pass # playlist specific folder try: os.mkdir('output/%s' % outdir) logging.info('output/%s created.' % outdir) except OSError: logging.info('output directory already exists, skipping creation.') self.ydl.params['outtmpl'] = 'output/' + outdir + '/%(title)s.%(ext)s' try: self.ydl.extract_info(url, download=True) except DownloadError as e: logging.info(e) return
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie._WORKING: print_skipping('IE marked as not _WORKING') return if 'playlist' not in test_case and not test_case['file']: print_skipping('No output file specified') return if 'skip' in test_case: print_skipping(test_case['skip']) return params = self.parameters.copy() params.update(test_case.get('params', {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.fd.add_progress_hook(_hook) test_cases = test_case.get('playlist', [test_case]) for tc in test_cases: _try_rm(tc['file']) _try_rm(tc['file'] + '.part') _try_rm(tc['file'] + '.info.json') try: for retry in range(1, RETRIES + 1): try: ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: if retry == RETRIES: raise # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError): raise print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry)) else: break for tc in test_cases: if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file']) self.assertTrue(tc['file'] in finished_hook_called) self.assertTrue(os.path.exists(tc['file'] + '.info.json')) if 'md5' in tc: md5_for_file = _file_md5(tc['file']) self.assertEqual(md5_for_file, tc['md5']) with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof: info_dict = json.load(infof) for (info_field, expected) in tc.get('info_dict', {}).items(): if isinstance(expected, compat_str) and expected.startswith('md5:'): got = 'md5:' + md5(info_dict.get(info_field)) else: got = info_dict.get(info_field) self.assertEqual(expected, got, u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # If checkable fields are missing from the test case, print the info_dict test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) for key, value in info_dict.items() if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location')) if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()): sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n') # Check for the presence of mandatory fields for key in ('id', 'url', 'title', 'ext'): self.assertTrue(key in info_dict.keys() and info_dict[key]) finally: for tc in test_cases: _try_rm(tc['file']) _try_rm(tc['file'] + '.part') _try_rm(tc['file'] + '.info.json')
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case["name"]) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get("add_ie", [])] def print_skipping(reason): print("Skipping %s: %s" % (test_case["name"], reason)) if not ie.working(): print_skipping("IE marked as not _WORKING") return if "playlist" not in test_case: info_dict = test_case.get("info_dict", {}) if not test_case.get("file") and not (info_dict.get("id") and info_dict.get("ext")): print_skipping( 'The output file cannot be know, the "file" ' "key is missing or the info_dict is incomplete" ) return if "skip" in test_case: print_skipping(test_case["skip"]) return for other_ie in other_ies: if not other_ie.working(): print_skipping(u"test depends on %sIE, marked as not WORKING" % other_ie.ie_key()) return params = get_params(test_case.get("params", {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status["status"] == "finished": finished_hook_called.add(status["filename"]) ydl.fd.add_progress_hook(_hook) def get_tc_filename(tc): return tc.get("file") or ydl.prepare_filename(tc.get("info_dict", {})) test_cases = test_case.get("playlist", [test_case]) def try_rm_tcs_files(): for tc in test_cases: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + ".part") try_rm(os.path.splitext(tc_filename)[0] + ".info.json") try_rm_tcs_files() try: try_num = 1 while True: try: ydl.download([test_case["url"]]) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or ( err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503 ): raise if try_num == RETRIES: report_warning(u"Failed due to network errors, skipping...") return print("Retrying: {0} failed tries\n\n##########\n\n".format(try_num)) try_num += 1 else: break for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get("params", {}).get("skip_download", False): self.assertTrue(os.path.exists(tc_filename), msg="Missing file " + tc_filename) self.assertTrue(tc_filename in finished_hook_called) info_json_fn = os.path.splitext(tc_filename)[0] + ".info.json" self.assertTrue(os.path.exists(info_json_fn)) if "md5" in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc["md5"]) with io.open(info_json_fn, encoding="utf-8") as infof: info_dict = json.load(infof) for (info_field, expected) in tc.get("info_dict", {}).items(): if isinstance(expected, compat_str) and expected.startswith("md5:"): got = "md5:" + md5(info_dict.get(info_field)) else: got = info_dict.get(info_field) self.assertEqual( expected, got, u"invalid value for field %s, expected %r, got %r" % (info_field, expected, got) ) # If checkable fields are missing from the test case, print the info_dict test_info_dict = dict( (key, value if not isinstance(value, compat_str) or len(value) < 250 else "md5:" + md5(value)) for key, value in info_dict.items() if value and key in ("title", "description", "uploader", "upload_date", "uploader_id", "location") ) if not all(key in tc.get("info_dict", {}).keys() for key in test_info_dict.keys()): sys.stderr.write( u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u"\n" ) # Check for the presence of mandatory fields for key in ("id", "url", "title", "ext"): self.assertTrue(key in info_dict.keys() and info_dict[key]) # Check for mandatory fields that are automatically set by YoutubeDL for key in ["webpage_url", "extractor", "extractor_key"]: self.assertTrue(info_dict.get(key), u"Missing field: %s" % key) finally: try_rm_tcs_files()
class NBC(object): def __init__(self): log('__init__') self.cache = SimpleCache() self.ydl = YoutubeDL() def openURL(self, url): log('openURL, url = ' + str(url)) try: cacheResponce = self.cache.get(ADDON_NAME + '.openURL, url = %s' % url) if not cacheResponce: request = urllib2.Request(url) responce = urllib2.urlopen(request, timeout=TIMEOUT).read() self.cache.set(ADDON_NAME + '.openURL, url = %s' % url, responce, expiration=datetime.timedelta(hours=1)) return self.cache.get(ADDON_NAME + '.openURL, url = %s' % url) except urllib2.URLError as e: log("openURL Failed! " + str(e), xbmc.LOGERROR) except socket.timeout as e: log("openURL Failed! " + str(e), xbmc.LOGERROR) except Exception as e: log("openURL Failed! " + str(e), xbmc.LOGERROR) xbmcgui.Dialog().notification(ADDON_NAME, LANGUAGE(30001), ICON, 4000) return '' def buildMenu(self, items): for item in items: self.addDir(*item) self.addYoutube("Browse Youtube", 'plugin://plugin.video.youtube/user/NBC/') def browseEpisodes(self, url=None): log('browseEpisodes') if url is None: url = VIDEO_URL + FILTER % ('type', 'Full%20Episode') items = json.loads(self.openURL(url)) if items and 'data' in items: for item in items['data']: path = item['attributes']['fullUrl'] aired = str(item['attributes']['airdate']).split('T')[0] duration = int(item['attributes']['runTime']) plot = item['attributes']['description'] title = item['attributes']['title'] showTitle = '' for show in item['attributes']['categories']: if show.startswith('Series'): showTitle = show.split('Series/')[1] break try: episodeNumber = int(item['attributes']['episodeNumber']) except: episodeNumber = 0 try: seasonNumber = int(item['attributes']['seasonNumber']) except: seasonNumber = 0 try: thumb = ICON for image in items['included']: if image['id'] == item['relationships']['image'][ 'data']['id']: thumb = BASE_URL + image['attributes']['path'] break except: thumb = ICON seinfo = ('S' + ('0' if seasonNumber < 10 else '') + str(seasonNumber) + 'E' + ('0' if episodeNumber < 10 else '') + str(episodeNumber)) label = '%s - %s' % ( showTitle, title ) if seasonNumber + episodeNumber == 0 else '%s - %s - %s' % ( showTitle, seinfo, title) infoLabels = { "mediatype": "episodes", "label": label, "title": label, "TVShowTitle": showTitle, "plot": plot, "aired": aired, "duration": duration, "season": seasonNumber, "episode": episodeNumber } infoArt = { "thumb": thumb, "poster": thumb, "fanart": FANART, "icon": ICON, "logo": ICON } self.addLink(label, path, 9, infoLabels, infoArt, len(items['data'])) try: next_page = items['links']['next'] except: next_page = None if next_page: self.addDir('>> Next', next_page, 1) def browseShows(self, url=None): log('browseShows') if url is None: url = SHOWS_URL items = json.loads(self.openURL(url)) if items and 'data' in items: for item in items['data']: showTitle = item['attributes']['shortTitle'] plot = (item['attributes']['shortDescription'] or showTitle).replace('<p>', '').replace('</p>', '') path = VIDEO_URL + FILTER % ('show', item['id']) vidID = item['relationships']['aggregates']['data']['id'] try: thumb = ICON for image in items['included']: if image['id'] == item['relationships']['image'][ 'data']['id']: thumb = BASE_URL + image['attributes']['path'] break except: thumb = ICON myURL = json.dumps({"url": path, "vidID": vidID}) infoLabels = { "mediatype": "tvshows", "label": showTitle, "title": showTitle, "TVShowTitle": showTitle, "plot": plot } infoArt = { "thumb": thumb, "poster": thumb, "fanart": FANART, "icon": ICON, "logo": ICON } self.addDir(showTitle, myURL, 0, infoLabels, infoArt) try: next_page = items['links']['next'] except: next_page = None if next_page: self.addDir('>> Next', next_page, 2) def resolveURL(self, url): log('resolveURL') myURL = json.loads(url) items = json.loads(self.openURL(SHOW_URL % myURL['vidID'])) if items and 'data' in items: for item in items['data']['attributes']['videoTypes']: self.browseEpisodes(myURL['url'] + FILTER % ('type', urllib2.quote(item))) def playVideo(self, name, url, liz=None): log('playVideo') self.ydl.add_default_info_extractors() with self.ydl: result = self.ydl.extract_info(url, download=False) url = result['manifest_url'] liz = xbmcgui.ListItem(name, path=url) xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, liz) def addYoutube(self, name, url): liz = xbmcgui.ListItem(name) liz.setProperty('IsPlayable', 'false') liz.setInfo(type="Video", infoLabels={"label": name, "title": name}) liz.setArt({'thumb': ICON, 'fanart': FANART}) xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=url, listitem=liz, isFolder=True) def addLink(self, name, u, mode, infoList=False, infoArt=False, total=0): name = name.encode("utf-8") log('addLink, name = ' + name) liz = xbmcgui.ListItem(name) liz.setProperty('IsPlayable', 'true') if infoList == False: liz.setInfo(type="Video", infoLabels={ "mediatype": "video", "label": name, "title": name, "genre": "News" }) else: liz.setInfo(type="Video", infoLabels=infoList) if infoArt == False: liz.setArt({'thumb': ICON, 'fanart': FANART}) else: liz.setArt(infoArt) u = sys.argv[0] + "?url=" + urllib.quote_plus(u) + "&mode=" + str( mode) + "&name=" + urllib.quote_plus(name) xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, totalItems=total) def addDir(self, name, u, mode, infoList=False, infoArt=False): name = name.encode("utf-8") log('addDir, name = ' + name) liz = xbmcgui.ListItem(name) liz.setProperty('IsPlayable', 'false') if infoList == False: liz.setInfo(type="Video", infoLabels={ "mediatype": "video", "label": name, "title": name, "genre": "News" }) else: liz.setInfo(type="Video", infoLabels=infoList) if infoArt == False: liz.setArt({'thumb': ICON, 'fanart': FANART}) else: liz.setArt(infoArt) u = sys.argv[0] + "?url=" + urllib.quote_plus(u) + "&mode=" + str( mode) + "&name=" + urllib.quote_plus(name) xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, isFolder=True)
def play_url(): # this only plays http urls for now, torrents soon. global title url = request.args.get('url') # grab url from /play?url=* if not url.startswith('http'): # in case the user forgot it log('url missing http/wrong protocol') url = 'http://' + url # let's assume it's http, not https log('received url %s' % url) log('requesting headers from %s...' % url) req = Request(url) req.get_method = lambda: 'HEAD' # only request headers, no content response = urlopen(req) ctype = response.headers['content-type'] ctype_split = ctype.split('/') # split into 2 parts log('headers received. content type is %s' % ctype) try: if ctype_split[0] == 'audio' or ctype_split[0] == 'video': log('url was raw media file, playing! :)') title = url # i guess this works? :T play_omxplayer(url) elif ctype_split[1] == 'x-bittorrent': log('loading torrents not implemented.') # this isn't implemented yet. elif ctype_split[0] == 'text': # here we check if it's a livestream, and if so get the RTMP url log('checking if url is a livestream...') live = Livestreamer() try: if "youtube" in url: raise RuntimeError( "youtube is f****d up w/ streaming, falling back to youtube-dl" ) plugin = live.resolve_url(url) streams = plugin.get_streams() stream = streams.get( "best") # fingers crossed for best quality stream_url_types = ['rtmp', 'url' ] # things that livestreamer can have :D for stream_type in stream_url_types: if hasattr(stream, stream_type): log('url is livestream!') title = "%s (livestream)" % url play_omxplayer(getattr(stream, stream_type)) return '', 204 except (PluginError, RuntimeError ) as e: # therefore url is not (supported) livestream pass # continue and let youtube-dl try. log('loading youtube-dl for further processing') ydl = YoutubeDL({ 'outtmpl': '%(id)s%(ext)s', 'restrictfilenames': True }) ydl.add_default_info_extractors() result = ydl.extract_info(url, download=False) if 'entries' in result: # if video is a playlist video = result['entries'][ 0] # play the 1st video in the playlist else: video = result play_omxplayer(video['url']) title = video['title'] else: raise DownloadError('Invalid filetype: not audio, video, or text.') return '', 204 # success w/ no response! except (UnicodeDecodeError, DownloadError) as e: return _ANSI_ESCAPE_REXP.sub('', str(e)), 400 # send error message
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return if 'playlist' not in test_case: info_dict = test_case.get('info_dict', {}) if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') if 'skip' in test_case: print_skipping(test_case['skip']) return for other_ie in other_ies: if not other_ie.working(): print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) return params = get_params(test_case.get('params', {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) test_cases = test_case.get('playlist', [test_case]) def try_rm_tcs_files(): for tc in test_cases: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 while True: try: ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): raise if try_num == RETRIES: report_warning(u'Failed due to network errors, skipping...') return print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) try_num += 1 else: break for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue(os.path.exists(info_json_fn)) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc['md5']) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, tc.get('info_dict', {}), info_dict) finally: try_rm_tcs_files()
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return if 'playlist' not in test_case: info_dict = test_case.get('info_dict', {}) if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')): print_skipping('The output file cannot be know, the "file" ' 'key is missing or the info_dict is incomplete') return if 'skip' in test_case: print_skipping(test_case['skip']) return for other_ie in other_ies: if not other_ie.working(): print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) return params = get_params(test_case.get('params', {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) test_cases = test_case.get('playlist', [test_case]) def try_rm_tcs_files(): for tc in test_cases: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 while True: try: ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): raise if try_num == RETRIES: report_warning(u'Failed due to network errors, skipping...') return print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) try_num += 1 else: break for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue(os.path.exists(info_json_fn)) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc['md5']) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) for (info_field, expected) in tc.get('info_dict', {}).items(): if isinstance(expected, compat_str) and expected.startswith('md5:'): got = 'md5:' + md5(info_dict.get(info_field)) else: got = info_dict.get(info_field) self.assertEqual(expected, got, u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # If checkable fields are missing from the test case, print the info_dict test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) for key, value in info_dict.items() if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location')) if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()): sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n') # Check for the presence of mandatory fields for key in ('id', 'url', 'title', 'ext'): self.assertTrue(key in info_dict.keys() and info_dict[key]) # Check for mandatory fields that are automatically set by YoutubeDL for key in ['webpage_url', 'extractor', 'extractor_key']: self.assertTrue(info_dict.get(key), u'Missing field: %s' % key) finally: try_rm_tcs_files()
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name'])() other_ies = [ get_info_extractor(ie_key)() for ie_key in test_case.get('add_ie', []) ] is_playlist = any(k.startswith('playlist') for k in test_case) test_cases = test_case.get('playlist', [] if is_playlist else [test_case]) def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) self.skipTest(reason) if not ie.working(): print_skipping('IE marked as not _WORKING') for tc in test_cases: info_dict = tc.get('info_dict', {}) if not (info_dict.get('id') and info_dict.get('ext')): raise Exception( 'Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file' % (tname, )) if 'skip' in test_case: print_skipping(test_case['skip']) for other_ie in other_ies: if not other_ie.working(): print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) params = get_params(test_case.get('params', {})) params['outtmpl'] = tname + '_' + params['outtmpl'] if is_playlist and 'playlist' not in test_case: params.setdefault('extract_flat', 'in_playlist') params.setdefault('playlistend', test_case.get('playlist_mincount')) params.setdefault('skip_download', True) ydl = YoutubeDL(params, auto_init=False) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) expect_warnings(ydl, test_case.get('expected_warnings', [])) def get_tc_filename(tc): return ydl.prepare_filename(tc.get('info_dict', {})) res_dict = None def try_rm_tcs_files(tcs=None): if tcs is None: tcs = test_cases for tc in tcs: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 while True: try: # We're not using .download here since that is just a shim # for outside error handling, and returns the exit code # instead of the result dict. res_dict = ydl.extract_info( test_case['url'], force_generic_extractor=params.get( 'force_generic_extractor', False)) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in ( compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or ( err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): msg = getattr(err, 'msg', error_to_compat_str(err)) err.msg = '%s (%s)' % ( msg, tname, ) raise err if try_num == RETRIES: report_warning( '%s failed due to network errors, skipping...' % tname) return print( 'Retrying: {0} failed tries\n\n##########\n\n'.format( try_num)) try_num += 1 else: break if is_playlist: self.assertTrue( res_dict['_type'] in ['playlist', 'multi_video']) self.assertTrue('entries' in res_dict) expect_info_dict(self, res_dict, test_case.get('info_dict', {})) if 'playlist_mincount' in test_case: assertGreaterEqual( self, len(res_dict['entries']), test_case['playlist_mincount'], 'Expected at least %d in playlist %s, but got only %d' % (test_case['playlist_mincount'], test_case['url'], len(res_dict['entries']))) if 'playlist_count' in test_case: self.assertEqual( len(res_dict['entries']), test_case['playlist_count'], 'Expected %d entries in playlist %s, but got %d.' % ( test_case['playlist_count'], test_case['url'], len(res_dict['entries']), )) if 'playlist_duration_sum' in test_case: got_duration = sum(e['duration'] for e in res_dict['entries']) self.assertEqual(test_case['playlist_duration_sum'], got_duration) # Generalize both playlists and single videos to unified format for # simplicity if 'entries' not in res_dict: res_dict['entries'] = [res_dict] for tc_num, tc in enumerate(test_cases): tc_res_dict = res_dict['entries'][tc_num] # First, check test cases' data against extracted data alone expect_info_dict(self, tc_res_dict, tc.get('info_dict', {})) # Now, check downloaded file consistency tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) expected_minsize = tc.get('file_minsize', 10000) if expected_minsize is not None: if params.get('test'): expected_minsize = max(expected_minsize, 10000) got_fsize = os.path.getsize(tc_filename) assertGreaterEqual( self, got_fsize, expected_minsize, 'Expected %s to be at least %s, but it\'s only %s ' % (tc_filename, format_bytes(expected_minsize), format_bytes(got_fsize))) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(tc['md5'], md5_for_file) # Finally, check test cases' data again but this time against # extracted data from info JSON file written during processing info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue(os.path.exists(info_json_fn), 'Missing info file %s' % info_json_fn) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, info_dict, tc.get('info_dict', {})) finally: try_rm_tcs_files() if is_playlist and res_dict is not None and res_dict.get( 'entries'): # Remove all other files that may have been extracted if the # extractor returns full results even with extract_flat res_tcs = [{'info_dict': e} for e in res_dict['entries']] try_rm_tcs_files(res_tcs)
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] is_playlist = any(k.startswith('playlist') for k in test_case) test_cases = test_case.get( 'playlist', [] if is_playlist else [test_case]) def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return for tc in test_cases: info_dict = tc.get('info_dict', {}) if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') if 'skip' in test_case: print_skipping(test_case['skip']) return for other_ie in other_ies: if not other_ie.working(): print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) return params = get_params(test_case.get('params', {})) if is_playlist and 'playlist' not in test_case: params.setdefault('extract_flat', True) params.setdefault('skip_download', True) ydl = YoutubeDL(params, auto_init=False) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) expect_warnings(ydl, test_case.get('expected_warnings', [])) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) res_dict = None def try_rm_tcs_files(tcs=None): if tcs is None: tcs = test_cases for tc in tcs: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 while True: try: # We're not using .download here sine that is just a shim # for outside error handling, and returns the exit code # instead of the result dict. res_dict = ydl.extract_info(test_case['url']) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): raise if try_num == RETRIES: report_warning(u'Failed due to network errors, skipping...') return print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) try_num += 1 else: break if is_playlist: self.assertEqual(res_dict['_type'], 'playlist') self.assertTrue('entries' in res_dict) expect_info_dict(self, test_case.get('info_dict', {}), res_dict) if 'playlist_mincount' in test_case: assertGreaterEqual( self, len(res_dict['entries']), test_case['playlist_mincount'], 'Expected at least %d in playlist %s, but got only %d' % ( test_case['playlist_mincount'], test_case['url'], len(res_dict['entries']))) if 'playlist_count' in test_case: self.assertEqual( len(res_dict['entries']), test_case['playlist_count'], 'Expected %d entries in playlist %s, but got %d.' % ( test_case['playlist_count'], test_case['url'], len(res_dict['entries']), )) if 'playlist_duration_sum' in test_case: got_duration = sum(e['duration'] for e in res_dict['entries']) self.assertEqual( test_case['playlist_duration_sum'], got_duration) for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) expected_minsize = tc.get('file_minsize', 10000) if expected_minsize is not None: if params.get('test'): expected_minsize = max(expected_minsize, 10000) got_fsize = os.path.getsize(tc_filename) assertGreaterEqual( self, got_fsize, expected_minsize, 'Expected %s to be at least %s, but it\'s only %s ' % (tc_filename, format_bytes(expected_minsize), format_bytes(got_fsize))) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc['md5']) info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue( os.path.exists(info_json_fn), 'Missing info file %s' % info_json_fn) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, tc.get('info_dict', {}), info_dict) finally: try_rm_tcs_files() if is_playlist and res_dict is not None and res_dict.get('entries'): # Remove all other files that may have been extracted if the # extractor returns full results even with extract_flat res_tcs = [{'info_dict': e} for e in res_dict['entries']] try_rm_tcs_files(res_tcs)
def process_url(self, source, global_opts): """Main method for processing urls This method basically hands over the configuration to YoutubeDL and repeats the step until every source in the configuration was read """ ydl = YoutubeDL() ydl.add_default_info_extractors() sourceUrl = source['url'] sourceDescription = source.get("description", "") self._logsource( "Processing source: '" + sourceDescription + "' Url: '" + sourceUrl + "'", source) # Merge local parameters with global ones ydl.params = copy.copy(global_opts) ydl.params.update(source) prefix = "" ydl.params['match_filter'] = (None if 'match_filter' not in ydl.params or ydl.params['match_filter'] is None else match_filter_func( ydl.params['match_filter'])) # Settings by commute tube over the standard settings, respect if the config sets them differently if 'format' not in ydl.params and 'format_limit' not in ydl.params: ydl.params[ 'format'] = "bestvideo+bestaudio/best" if 'format' not in self.config else self.config[ "format"] if 'nooverwrites' not in ydl.params: ydl.params['nooverwrites'] = True if 'ignoreerrors' not in ydl.params: ydl.params['ignoreerrors'] = True if 'download_archive' not in ydl.params: ydl.params['download_archive'] = self.download_archive if 'prefix' in ydl.params: prefix = ydl.params['prefix'] ydl.params['restrictfilenames'] = True ydl.params['logger'] = self.ydlLog outtmpl = self.pathToDownloadFolder + "/" + prefix + \ '%(uploader)s-%(title)s.%(ext)s' if 'outtmpl' not in ydl.params: ydl.params['outtmpl'] = outtmpl elif not (ydl.params['outtmpl'].startswith(self.pathToDownloadFolder)): self._logsource( "Prefixing custom set outtmpl with '" + self.pathToDownloadFolder + "/" + prefix + "'", source) ydl.params['outtmpl'] = self.pathToDownloadFolder + "/" + prefix + \ ydl.params['outtmpl'] if self.debug: self._logsource( "All downloads will be simulated since this is debug mode", source) ydl.params['simulate'] = True ydl.download([source['url']])
"no_warnings": True, "outtmpl": "%(title)s.%(ext)s", "download_archive": "download_archive.txt", "ignoreerrors": True, # ignore errors "writethumbnail": True, "writesubtitles": True, "allsubtitles": True, "min_sleep_interval": 5, "max_sleep_interval": 15, "addmetadata": True, # 'listformats': True, # print a list of the formats to stdout and exit # "forcejson": True, } ydl = YoutubeDL(ydl_opts) ydl.add_default_info_extractors() headers = { "User-Agent": "Mozilla/5.0 (Linux; Android 7.1.2; SM-G610M) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/71.0.3578.99 Mobile Safari/537.36" } PUNCT_TO_REMOVE = re.sub('[/-]', '', string.punctuation) # Keep dashes and forward slashes translator = str.maketrans("", "", PUNCT_TO_REMOVE) tvdb_key = '1DE5C9B35180B706' t = tvdb_api.Tvdb(apikey=tvdb_key)
from threading import Timer import json import logging import time import re import random from youtube_dl import YoutubeDL from youtube_dl.utils import DownloadError from minilodon.minilodon import Minilodon bot = Minilodon("config.json") logger = logging.getLogger(__name__) ydl = YoutubeDL({'quiet': False, 'logger': logger, 'noplaylist': True, 'extract_flat': 'in_playlist'}) ydl.add_default_info_extractors() spy_function = None spy_timer = None roll_regex = re.compile(r"([0-9]*)d([0-9]+)") @bot.command("update", True) def update(nick, args): if len(args) < 4: yield "Usage: !update <category> <key> <msg>" return category = args[1].lower() key = args[2].lower() msg = " ".join(args[3:]) if len(msg) > 254: yield "Warning: Entry too long, message will be wrapped." try:
from youtube_dl import YoutubeDL from .omx_handler import OmxRequestHandler _ytdl = YoutubeDL(dict(forceurl=True, skip_download=True, restrictfilenames=False)) _ytdl.add_default_info_extractors() class YtDlHandler(OmxRequestHandler): def get_location(self): self.set_header("Access-Control-Allow-Origin", "*") loc = self.get_argument("location") info = _ytdl.extract_info(loc, download=False) return info["url"]
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie._WORKING: print_skipping('IE marked as not _WORKING') return if 'playlist' not in test_case and not test_case['file']: print_skipping('No output file specified') return if 'skip' in test_case: print_skipping(test_case['skip']) return params = get_params(test_case.get('params', {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.fd.add_progress_hook(_hook) test_cases = test_case.get('playlist', [test_case]) for tc in test_cases: try_rm(tc['file']) try_rm(tc['file'] + '.part') try_rm(tc['file'] + '.info.json') try: for retry in range(1, RETRIES + 1): try: ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: if retry == RETRIES: raise # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError): raise print( 'Retrying: {0} failed tries\n\n##########\n\n'.format( retry)) else: break for tc in test_cases: if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file']) self.assertTrue(tc['file'] in finished_hook_called) self.assertTrue(os.path.exists(tc['file'] + '.info.json')) if 'md5' in tc: md5_for_file = _file_md5(tc['file']) self.assertEqual(md5_for_file, tc['md5']) with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof: info_dict = json.load(infof) for (info_field, expected) in tc.get('info_dict', {}).items(): if isinstance(expected, compat_str) and expected.startswith('md5:'): got = 'md5:' + md5(info_dict.get(info_field)) else: got = info_dict.get(info_field) self.assertEqual( expected, got, u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # If checkable fields are missing from the test case, print the info_dict test_info_dict = dict( (key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) for key, value in info_dict.items() if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location')) if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()): sys.stderr.write(u'\n"info_dict": ' + json.dumps( test_info_dict, ensure_ascii=False, indent=2) + u'\n') # Check for the presence of mandatory fields for key in ('id', 'url', 'title', 'ext'): self.assertTrue(key in info_dict.keys() and info_dict[key]) finally: for tc in test_cases: try_rm(tc['file']) try_rm(tc['file'] + '.part') try_rm(tc['file'] + '.info.json')
def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [ get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', []) ] def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return if 'playlist' not in test_case: info_dict = test_case.get('info_dict', {}) if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')): raise Exception( 'Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?' ) if 'skip' in test_case: print_skipping(test_case['skip']) return for other_ie in other_ies: if not other_ie.working(): print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) return params = get_params(test_case.get('params', {})) ydl = YoutubeDL(params) ydl.add_default_info_extractors() finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) ydl.add_progress_hook(_hook) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename( tc.get('info_dict', {})) test_cases = test_case.get('playlist', [test_case]) def try_rm_tcs_files(): for tc in test_cases: tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 while True: try: ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in ( compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or ( err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): raise if try_num == RETRIES: report_warning( u'Failed due to network errors, skipping...') return print( 'Retrying: {0} failed tries\n\n##########\n\n'.format( try_num)) try_num += 1 else: break for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue(os.path.exists(info_json_fn)) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc['md5']) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, tc.get('info_dict', {}), info_dict) finally: try_rm_tcs_files()