def __init__(self,
logger,
ydl_params=None,
f=YDL_LOADER_FORMAT.MP3,
base_url='https://www.youtube.com/watch'):
self._base_url = base_url
ydl = youtube_dl.YoutubeDL({'listsubtitles': True, 'logger': logger})
self._video_descr_extractor = ydl.get_info_extractor(
youtube_dl.gen_extractors()[1125].ie_key())
audio_ydl_params = ydl_params
if audio_ydl_params is None:
audio_ydl_params = {
'writeautomaticsub': True,
'outtmpl':
'data/videos/youtube/%(channel_id)s/%(id)s.' + f.value,
'format': 'bestaudio/best',
'prefer-avconv': True,
'subtitleslangs': ['ru'],
'ext': f.value,
'simulate': False,
'max_sleep_interval': 2,
'sleep_interval': 1,
'ignoreerrors': False,
}
audio_ydl_params['logger'] = logger
self._audio_ydl = youtube_dl.YoutubeDL(audio_ydl_params)
def main():
parser = optparse.OptionParser(usage='%prog OUTFILE.md')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('Expected an output filename')
outfile, = args
def gen_ies_md(ies):
for ie in ies:
ie_md = '**{0}**'.format(ie.IE_NAME)
ie_desc = getattr(ie, 'IE_DESC', None)
if ie_desc is False:
continue
if ie_desc is not None:
ie_md += ': {0}'.format(ie.IE_DESC)
if not ie.working():
ie_md += ' (Currently broken)'
yield ie_md
ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower())
out = '# Supported sites\n' + ''.join(
' - ' + md + '\n'
for md in gen_ies_md(ies))
with io.open(outfile, 'w', encoding='utf-8') as outf:
outf.write(out)
def _get_stream_path(self):
"""Check if video stream comes from a video sharing platform"""
if not youtube_dl_available():
return self.path
import youtube_dl
ydl_options = {
'format': self.ydl_format,
'quiet': True,
'no_warnings': True
}
for extractor in youtube_dl.gen_extractors():
if extractor.suitable(self.path):
try:
with youtube_dl.YoutubeDL(ydl_options) as ydl:
stream_info = ydl.extract_info(self.path,
download=False)
self.ydl_info = stream_info
if stream_info['url'] is None:
raise ValueError(
'Unable to find stream URL for video format '
f'{self.ydl_format}')
return stream_info['url']
except youtube_dl.utils.YoutubeDLError:
break
return self.path
def start(self):
try:
# omit the last extractor (GenericIE, which takes forever)
extractors = youtube_dl.gen_extractors()[:-1]
youtube_dl.gen_extractors = lambda: extractors
if '_ALL_CLASSES' in youtube_dl.extractor.__dict__:
youtube_dl.extractor._ALL_CLASSES = youtube_dl.extractor._ALL_CLASSES[:
-1]
# set up redirection
self.devnull = open(os.devnull, 'w')
self.noexit = lambda m: None
ydl_prepare_filename = youtube_dl.YoutubeDL.prepare_filename
this = self
def prepare_filename(self, info_dict):
filename = ydl_prepare_filename(self, info_dict)
info_dict['_filename'] = filename
this.info_dicts.append(info_dict)
return filename
youtube_dl.YoutubeDL.prepare_filename = prepare_filename
self.started = True
except:
if self in scrapers: scrapers.remove(self)
sys.stderr.write('unable to start youtube-dl\n')
traceback.print_exc()
def __init__(self, path, log, echo=False):
engine = create_engine(path, echo=echo)
Base.metadata.create_all(engine)
session = sessionmaker()
session.configure(bind=engine)
self.session = session()
self.extractors = gen_extractors()
self.converters = gen_converters()
self.log = log
version = self.__select_config('version').first()
if version is None or version < Database.__version__:
self.session.merge(Config(
id='version',
value=Database.__version__
))
self.session.commit()
def main():
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
template = tmplf.read()
ie_htmls = []
for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME):
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
try:
ie_html += ': {}'.format(ie.IE_DESC)
except AttributeError:
pass
if ie.working() == False:
ie_html += ' (Currently broken)'
ie_htmls.append('<li>{}</li>'.format(ie_html))
template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
sitesf.write(template)
def main():
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
template = tmplf.read()
ie_htmls = []
for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()):
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
ie_desc = getattr(ie, 'IE_DESC', None)
if ie_desc is False:
continue
elif ie_desc is not None:
ie_html += ': {}'.format(ie.IE_DESC)
if not ie.working():
ie_html += ' (Currently broken)'
ie_htmls.append('<li>{}</li>'.format(ie_html))
template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
sitesf.write(template)
def _source(url, ie_key, title, info):
if info['id']:
if ie_key == 'Youtube':
return '{{From YouTube|1=%(id)s|2=%(title)s}}' % \
{'id': info['id'], 'title': escape_wikitext(title)}
elif ie_key == 'Vimeo':
return '{{From Vimeo|1=%(id)s|2=%(title)s}}' % \
{'id': info['id'], 'title': escape_wikitext(title)}
if ie_key == 'Generic':
return url
else:
if ':' in info['extractor']:
# Try to find the anme of the 'owner' of this sub-ie
ie_tmp = info['extractor'][:info['extractor'].index(':')]
for ie in youtube_dl.gen_extractors():
if ie.IE_NAME == ie_tmp:
ie_key = ie.ie_key()
break
return '[%(url)s %(title)s - %(extractor)s]' % \
{'url': url, 'title': escape_wikitext(title), 'extractor': ie_key}
def main():
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
template = tmplf.read()
ie_htmls = []
for ie in sorted(youtube_dl.gen_extractors(),
key=lambda i: i.IE_NAME.lower()):
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
ie_desc = getattr(ie, 'IE_DESC', None)
if ie_desc is False:
continue
elif ie_desc is not None:
ie_html += ': {}'.format(ie.IE_DESC)
if not ie.working():
ie_html += ' (Currently broken)'
ie_htmls.append('<li>{}</li>'.format(ie_html))
template = template.replace('@SITES@',
textwrap.indent('\n'.join(ie_htmls), '\t'))
with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
sitesf.write(template)
def start(self):
try:
# omit the last extractor (GenericIE, which takes forever)
extractors = youtube_dl.gen_extractors()[:-1]
youtube_dl.gen_extractors = lambda:extractors
if '_ALL_CLASSES' in youtube_dl.extractor.__dict__:
youtube_dl.extractor._ALL_CLASSES = youtube_dl.extractor._ALL_CLASSES[:-1]
# set up redirection
self.devnull = open(os.devnull, 'w')
self.noexit = lambda m:None
ydl_prepare_filename = youtube_dl.YoutubeDL.prepare_filename
this = self
def prepare_filename(self, info_dict):
filename = ydl_prepare_filename(self, info_dict)
info_dict['_filename'] = filename
this.info_dicts.append(info_dict)
return filename
youtube_dl.YoutubeDL.prepare_filename = prepare_filename
self.started = True
except:
if self in scrapers: scrapers.remove(self)
sys.stderr.write('unable to start youtube-dl\n')
traceback.print_exc()
Created on 27.08.2014
@author: ole
'''
import youtube_dl
import argparse
import urllib2
import re
import random
import string
import difflib
supported = list()
for extractor in youtube_dl.gen_extractors():
name = extractor.IE_NAME
supported.append(name.lower())
#supported contains a list of supported websites by youtube-dl
parser = argparse.ArgumentParser(description='Download TV-series.', epilog="If you don't provide an episode this"
+ "program will try to download the provided season. If you don't provide a season it"
+"will try to download all seasons.")
parser.add_argument("name", help="name of the series you want to download", nargs=1)
parser.add_argument("-s", metavar="season", dest="season", help="the number of the season you want to download",
type=int, nargs=1, required=False)
parser.add_argument("-e", metavar="episode", dest="episode", help="the number of the episode you want to download",
type=int, nargs=1, required=False)
def list_extractors():
ie_list = [{
'name': ie.IE_NAME,
'working': ie.working(),
} for ie in youtube_dl.gen_extractors()]
return jsonify(extractors=ie_list)
def on_feed_output(self, feed, config):
jar = cookielib.CookieJar()
cookie_processor = urllib2.HTTPCookieProcessor(jar)
proxy_handler = urllib2.ProxyHandler()
opener = urllib2.build_opener(proxy_handler, cookie_processor, youtube_dl.YoutubeDLHandler())
urllib2.install_opener(opener)
for entry in feed.accepted:
if entry.get('urls'):
urls = entry.get('urls')
else:
urls = [entry['url']]
pubdate = entry.get('rss_pubdate')
errors = []
if not feed.manager.options.test:
fd = youtube_dl.FileDownloader({
'usenetrc': None,
'username': None,
'password': None,
'quiet': False,
'forceurl': False,
'forcetitle': False,
'forcethumbnail': False,
'forcedescription': False,
'forcefilename': False,
'forceformat': False,
'simulate': False,
'skip_download': False,
'format': None,
'format_limit': None,
'listformats': None,
'outtmpl': (config.get('directory') and u'%(directory)s/%(year)04d-%(month)02d-%(day)02d-%%(id)s.%%(ext)s' % {'directory': config.get('directory'), 'year': pubdate.year, 'month': pubdate.month, 'day': pubdate.day})
or config.get('template'),
'ignoreerrors': False,
'ratelimit': None,
'nooverwrites': False,
'retries': 10,
'continuedl': True,
'noprogress': False,
'playliststart': 1,
'playlistend': -1,
'logtostderr': False,
'consoletitle': False,
'nopart': False,
'updatetime': True,
'writedescription': False,
'writeinfojson': False,
'writesubtitles': False,
'subtitleslang': None,
'matchtitle': None,
'rejecttitle': None,
'max_downloads': None,
'prefer_free_formats': False,
'verbose': False,
})
for extractor in youtube_dl.gen_extractors():
fd.add_info_extractor(extractor)
urls = map(lambda url: url.encode('utf-8').strip(), urls)
fd.download(urls)
urllib2.install_opener(urllib2.build_opener())
def list_extractors():
ie_list = [{
'name': ie.IE_NAME,
'working': ie.working(),
} for ie in youtube_dl.gen_extractors()]
return jsonify(extractors=ie_list)
async def extractors():
ie_list = [{
'name': ie.IE_NAME,
'working': ie.working(),
} for ie in youtube_dl.gen_extractors()]
return dict(extractors=ie_list)
def list_extractors():
ie_list = [{"name": ie.IE_NAME, "working": ie.working()} for ie in youtube_dl.gen_extractors()]
return jsonify(extractors=ie_list)
def get_response(self):
return [dict_for_IE(ie) for ie in gen_extractors()]
