class Sportschau(Plugin):
_re_player = re.compile(
r"https?:(//deviceids-medp.wdr.de/ondemand/\S+\.js)")
_re_json = re.compile(
r"\$mediaObject.jsonpHelper.storeAndPlay\(({.+})\);?")
_schema_player = validate.Schema(
validate.transform(_re_player.search),
validate.any(
None,
validate.Schema(
validate.get(1),
validate.transform(lambda url: update_scheme("https:", url)))))
_schema_json = validate.Schema(
validate.transform(_re_json.match), validate.get(1),
validate.transform(parse_json), validate.get("mediaResource"),
validate.get("dflt"), validate.get("videoURL"),
validate.transform(lambda url: update_scheme("https:", url)))
def _get_streams(self):
player_js = self.session.http.get(self.url, schema=self._schema_player)
if not player_js:
return
log.debug("Found player js {0}".format(player_js))
hls_url = self.session.http.get(player_js, schema=self._schema_json)
yield from HLSStream.parse_variant_playlist(self.session,
hls_url).items()
def _get_streams(self):
m = self._live_url_re.search(self.page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
player_url = self.url
token = self._get_token()
if playlist_url:
log.debug("Found playlist URL in the page")
else:
live_channel = None
for div in itertags(self.page.text, "div"):
if div.attributes.get("id") == "botonLive":
live_channel = div.attributes.get("data-canal")
if live_channel:
log.debug("Live channel: {0}".format(live_channel))
player_url = self._channel_urls[live_channel] + quote(token)
page = self.session.http.get(player_url,
raise_for_status=False)
if "block access from your country." in page.text:
raise PluginError("Content is geo-locked")
m = self._playlist_re.search(page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
else:
log.error("Could not find the live channel")
if playlist_url:
stream_url = "{0}?{1}".format(playlist_url,
urlencode({"iut": token}))
return HLSStream.parse_variant_playlist(
self.session, stream_url, headers={"referer": player_url})
def _get_streams(self):
http.headers.update({'User-Agent': useragents.FIREFOX})
log.debug('Version 2018-07-01')
log.info('This is a custom plugin. '
'For support visit https://github.com/back-to/plugins')
res = http.get(self.url)
data = self._data_re.search(res.text)
if data:
log.debug('Found _data_re')
data = self.js_to_json_regex(data.group(1))
res = http.post(self.api_url, data=data)
m = self._hls_re.search(res.text)
if m:
log.debug('Found _hls_re')
hls_url = m.group('url')
hls_url = update_scheme('http://', hls_url)
log.debug('URL={0}'.format(hls_url))
streams = HLSStream.parse_variant_playlist(
self.session, hls_url)
if not streams:
return {'live': HLSStream(self.session, hls_url)}
else:
return streams
iframe = self._iframe_re.search(res.text)
if iframe:
log.debug('Found _iframe_re')
iframe_url = iframe.group('url')
iframe_url = update_scheme('http://', iframe_url)
log.debug('URL={0}'.format(iframe_url))
return self.session.streams(iframe_url)
def test_update_scheme(self):
self.assertEqual(
"https://example.com/foo",
update_scheme("https://other.com/bar", "//example.com/foo"))
self.assertEqual(
"http://example.com/foo",
update_scheme("http://other.com/bar", "//example.com/foo"))
self.assertEqual(
"http://example.com/foo",
update_scheme("https://other.com/bar", "http://example.com/foo"))
def test_update_scheme(self):
self.assertEqual(
"https://example.com/foo", # becomes https
update_scheme("https://other.com/bar", "//example.com/foo"))
self.assertEqual(
"http://example.com/foo", # becomes http
update_scheme("http://other.com/bar", "//example.com/foo"))
self.assertEqual(
"http://example.com/foo", # remains unchanged
update_scheme("https://other.com/bar", "http://example.com/foo"))
self.assertEqual(
"https://example.com/foo", # becomes https
update_scheme("https://other.com/bar", "example.com/foo"))
def _get_streams(self):
self.session.http.headers.update({
'Referer':
'http://www.abweb.com/BIS-TV-Online/bistvo-tele-universal.aspx'
})
login_username = self.get_option('username')
login_password = self.get_option('password')
if self.options.get('purge_credentials'):
self.clear_cookies()
self._authed = False
log.info('All credentials were successfully removed.')
if self._authed:
log.info('Attempting to authenticate using cached cookies')
elif not self._authed and not (login_username and login_password):
log.error(
'A login for ABweb is required, use --abweb-username USERNAME --abweb-password PASSWORD'
)
return
elif not self._authed and not self._login(login_username,
login_password):
return
log.debug('get iframe_url')
res = self.session.http.get(self.url)
for iframe in itertags(res.text, 'iframe'):
iframe_url = iframe.attributes.get('src')
if iframe_url.startswith('/'):
iframe_url = url_concat('https://www.abweb.com', iframe_url)
else:
iframe_url = update_scheme('https://', iframe_url)
log.debug('iframe_url={0}'.format(iframe_url))
break
else:
raise PluginError('No iframe_url found.')
self.session.http.headers.update({'Referer': iframe_url})
res = self.session.http.get(iframe_url)
m = self._hls_re.search(res.text)
if not m:
raise PluginError('No hls_url found.')
hls_url = update_scheme('https://', m.group('url'))
streams = HLSStream.parse_variant_playlist(self.session, hls_url)
if streams:
for stream in streams.items():
yield stream
else:
yield 'live', HLSStream(self.session, hls_url)
def _get_streams(self):
"""
Find the streams for web.tv
:return:
"""
headers = {}
res = self.session.http.get(self.url, headers=headers)
headers["Referer"] = self.url
sources = self._sources_re.findall(res.text)
if len(sources):
sdata = parse_json(sources[0], schema=self._sources_schema)
for source in sdata:
self.logger.debug("Found stream of type: {}", source[u'type'])
if source[u'type'] == u"application/vnd.apple.mpegurl":
url = update_scheme(self.url, source[u"src"])
try:
# try to parse the stream as a variant playlist
variant = HLSStream.parse_variant_playlist(self.session, url, headers=headers)
if variant:
for q, s in variant.items():
yield q, s
else:
# and if that fails, try it as a plain HLS stream
yield 'live', HLSStream(self.session, url, headers=headers)
except IOError:
self.logger.warning("Could not open the stream, perhaps the channel is offline")
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
iframe_url = None
page = self.session.http.get(self.url)
for a in itertags(page.text, 'a'):
if a.attributes.get('class') == 'play-live':
iframe_url = update_scheme(self.url, a.attributes['data-url'])
break
if not iframe_url:
raise PluginError('Could not find iframe.')
parsed = urlparse(iframe_url)
path_list = parsed.path.split('/')
if len(path_list) != 6:
# only support a known iframe url style,
# the video id might be on a different spot if the url changes
raise PluginError('unsupported iframe URL: {0}'.format(iframe_url))
res = self.session.http.get(
self.API_URL.format(netloc=parsed.netloc, id=path_list[4]))
data = self.session.http.json(res, schema=self._api_schema)
log.trace('{0!r}'.format(data))
url = self.PLAYLIST_URL.format(
app=data['streamProperties']['application'],
name=data['playStreamName'],
netloc=data['cdnHost'],
)
return HLSStream.parse_variant_playlist(self.session, url)
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme("http://", urlnoproto)
return HDSStream.parse_manifest(self.session, urlnoproto, **params)
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
log.debug('Version 2018-07-12')
log.info('This is a custom plugin. '
'For support visit https://github.com/back-to/plugins')
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme('http://', urlnoproto)
streams = self.session.streams(urlnoproto, stream_types=['hls'])
if not streams:
log.debug('No stream found for hls-key-uri,'
' stream is not available.')
return
stream = streams['best']
urlnoproto = stream.url
log.debug('URL={0}; params={1}', urlnoproto, params)
streams = KeyUriHLSStream.parse_variant_playlist(
self.session, urlnoproto, **params)
if not streams:
return {
'live': KeyUriHLSStream(self.session, urlnoproto, **params)
}
else:
return streams
def _get_streams(self):
if self.is_live:
log.debug("Loading live stream for {0}...".format(self.channel))
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
log.debug("Attempting to find VOD stream for {0}...".format(self.channel))
vod_data = self.vod_data()
if vod_data:
log.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_streams(self):
"""
Find the streams for web.tv
:return:
"""
headers = {}
res = self.session.http.get(self.url, headers=headers)
headers["Referer"] = self.url
sources = self._sources_re.findall(res.text)
if len(sources):
sdata = parse_json(sources[0], schema=self._sources_schema)
for source in sdata:
self.logger.debug("Found stream of type: {}", source[u'type'])
if source[u'type'] == u"application/vnd.apple.mpegurl":
url = update_scheme(self.url, source[u"src"])
try:
# try to parse the stream as a variant playlist
variant = HLSStream.parse_variant_playlist(
self.session, url, headers=headers)
if variant:
for q, s in variant.items():
yield q, s
else:
# and if that fails, try it as a plain HLS stream
yield 'live', HLSStream(self.session,
url,
headers=headers)
except IOError:
self.logger.warning(
"Could not open the stream, perhaps the channel is offline"
)
def _get_streams(self):
data = self.match.groupdict()
url = update_scheme("http://", data.get("url"))
params = parse_params(data.get("params"))
log.debug(f"URL={url}; params={params}")
return {"live": AkamaiHDStream(self.session, url, **params)}
def _get_streams(self):
if self.is_live:
self.logger.debug("Loading live stream for {0}...", self.channel)
res = http.get(self.live_api_url,
data={"r": random.randint(1, 100000)})
live_data = http.json(res)
for url in live_data.get(
"hls",
[])[:1]: # only take the first, they are all the same streams
url = url + '&' + urlencode(
self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(
self.session, url,
name_fmt="{pixels}_{bitrate}").items():
yield s
elif self.channel == "1tv":
self.logger.debug("Attempting to find VOD stream...", self.channel)
vod_data = self.vod_data()
if vod_data:
self.logger.info(u"Found VOD: {0}".format(
vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(
self.session, update_scheme(self.url, stream['src']))
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme("http://", urlnoproto)
return HDSStream.parse_manifest(self.session, urlnoproto, **params)
def _get_streams(self):
res = self.session.http.get(self.url)
match = _player_js.search(res.text)
if match:
player_js = match.group(0)
self.logger.info("Found player js {0}", player_js)
else:
self.logger.info("Didn't find player js. Probably this page doesn't contain a video")
return
res = self.session.http.get(player_js)
jsonp_start = res.text.find('(') + 1
jsonp_end = res.text.rfind(')')
if jsonp_start <= 0 or jsonp_end <= 0:
self.logger.info("Couldn't extract json metadata from player.js: {0}", player_js)
return
json_s = res.text[jsonp_start:jsonp_end]
stream_metadata = json.loads(json_s)
hds_url = stream_metadata['mediaResource']['dflt']['videoURL']
hds_url = update_scheme(self.url, hds_url)
return HDSStream.parse_manifest(self.session, hds_url).items()
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
iframe_url = None
page = self.session.http.get(self.url)
for a in itertags(page.text, 'a'):
if a.attributes.get('class') == 'play-live':
iframe_url = update_scheme(self.url, a.attributes['data-url'])
break
if not iframe_url:
raise PluginError('Could not find iframe.')
parsed = urlparse(iframe_url)
path_list = parsed.path.split('/')
if len(path_list) != 6:
# only support a known iframe url style,
# the video id might be on a different spot if the url changes
raise PluginError('unsupported iframe URL: {0}'.format(iframe_url))
res = self.session.http.get(
self.API_URL.format(netloc=parsed.netloc, id=path_list[4]))
data = self.session.http.json(res, schema=self._api_schema)
log.trace('{0!r}'.format(data))
url = self.PLAYLIST_URL.format(
app=data['streamProperties']['application'],
name=data['playStreamName'],
netloc=data['cdnHost'],
)
return HLSStream.parse_variant_playlist(self.session, url)
def find_iframe(self, url):
self.session.http.headers.update({"User-Agent": useragents.CHROME})
res = self.session.http.get(self.url)
for iframe_url in self.iframe_re.findall(res.text):
if "googletagmanager" not in iframe_url:
iframe_url = html_unescape(iframe_url)
return update_scheme(self.url, iframe_url)
def _get_streams(self):
res = self.session.http.get(self.url)
match = _player_js.search(res.text)
if match:
player_js = match.group(0)
self.logger.info("Found player js {0}", player_js)
else:
self.logger.info(
"Didn't find player js. Probably this page doesn't contain a video"
)
return
res = self.session.http.get(player_js)
jsonp_start = res.text.find('(') + 1
jsonp_end = res.text.rfind(')')
if jsonp_start <= 0 or jsonp_end <= 0:
self.logger.info(
"Couldn't extract json metadata from player.js: {0}",
player_js)
return
json_s = res.text[jsonp_start:jsonp_end]
stream_metadata = json.loads(json_s)
hds_url = stream_metadata['mediaResource']['dflt']['videoURL']
hds_url = update_scheme(self.url, hds_url)
return HDSStream.parse_manifest(self.session, hds_url).items()
def _get_live_streams(self):
# Get channel id
match = self._url_re.match(self.url)
channel = match.group('channel')
# Retrieve live player URL
res = http.get(self.PLAYER_URL)
match = self._live_player_re.search(res.text)
if match is None:
return []
live_player_url = update_scheme(self.url,
match.group('live_player_url'))
# Extract streams from the live player page
res = http.get(live_player_url)
stream_datas = re.findall(
r'{0}(?:_MINI)?:({{.+?}}]}}]}})'.format(self.CHANNEL_MAP[channel]),
res.text)
streams = []
for s in stream_datas:
for u in self._live_streams_schema.validate(s):
if u not in streams:
streams.append(u)
return streams
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(1)
urlnoproto = update_scheme("http://", urlnoproto)
log.debug("URL={0}; params={1}".format(urlnoproto, params))
return {"live": AkamaiHDStream(self.session, urlnoproto, **params)}
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(1)
urlnoproto = update_scheme("http://", urlnoproto)
self.logger.debug("URL={0}; params={1}", urlnoproto, params)
return {"live": HTTPStream(self.session, urlnoproto, **params)}
def _find_video_id(self, url):
m = _url_re.match(url)
if m and m.group("video_id"):
log.debug("Video & PL ID from URL")
return m.group("video_id"), m.group("pl_id")
res = self.session.http.get(
update_scheme("https://", url.replace("youtubedl://", "")))
datam = _ytdata_re.search(res.text)
if datam:
data = parse_json(datam.group(1))
# find the videoRenderer object, where there is a LIVE NOW badge
for vid_ep in search_dict(data, "currentVideoEndpoint"):
video_id = vid_ep.get("watchEndpoint", {}).get("videoId")
if video_id:
log.debug("Video ID from currentVideoEndpoint")
return video_id, None
for x in search_dict(data, "videoRenderer"):
for bstyle in search_dict(x.get("badges", {}), "style"):
if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
if x.get("videoId"):
log.debug("Video ID from videoRenderer (live)")
return x["videoId"], None
if "/embed/live_stream" in url:
for link in itertags(res.text, "link"):
if link.attributes.get("rel") == "canonical":
canon_link = link.attributes.get("href")
if canon_link != url:
log.debug("Re-directing to canonical URL: {0}".format(
canon_link))
return self._find_video_id(canon_link), None
raise PluginError("Could not find a Video ID in URL")
def _get_streams(self):
res = self.session.http.get(self.url)
# Look for Youtube embedded video first
for iframe in itertags(res.text, "iframe"):
if urlparse(iframe.attributes.get("src")).netloc.endswith(
"youtube.com"):
log.debug("Handing off to YouTube plugin")
return self.session.streams(iframe.attributes.get("src"))
# Next check for HLS URL with token
mobile_url_m = self.mobile_url_re.search(res.text)
mobile_url = mobile_url_m and update_scheme(self.url,
mobile_url_m.group("url"))
if mobile_url:
log.debug("Found mobile stream: {0}".format(mobile_url_m.group(0)))
token = mobile_url_m and mobile_url_m.group("token")
if not token and "kralmuzik" in self.url:
log.debug("Getting Kral Muzik HLS stream token from API")
token = self.session.http.get(self.kral_token_url).text
elif not token:
# if no token is in the url, try to find it else where in the page
log.debug("Searching for HLS stream token in URL")
token_m = self.token_re.search(res.text)
token = token_m and token_m.group("token")
return HLSStream.parse_variant_playlist(
self.session,
mobile_url + token,
headers={"Referer": self.url})
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(1)
urlnoproto = update_scheme("http://", urlnoproto)
self.logger.debug("URL={0}; params={1}", urlnoproto, params)
return {"live": AkamaiHDStream(self.session, urlnoproto, **params)}
def _get_streams(self):
res = self.session.http.get(self.url)
match = _media_id_re.search(res.text)
if match:
media_id = match.group(1)
else:
return
log.debug("Found media id: {0}".format(media_id))
res = self.session.http.get(MEDIA_URL.format(media_id))
media = self.session.http.json(res, schema=_media_schema)
log.trace("{0!r}".format(media))
for media in media["_mediaArray"]:
for stream in media["_mediaStreamArray"]:
stream_ = stream["_stream"]
if isinstance(stream_, list):
if not stream_:
continue
stream_ = stream_[0]
stream_ = update_scheme("https://", stream_)
if ".m3u8" in stream_:
parser = self._get_hls_streams
parser_name = "HLS"
elif (".mp4" in stream_ and ".f4m" not in stream_):
parser = self._get_http_streams
parser_name = "HTTP"
else:
log.error("Unexpected stream type: '{0}'".format(stream_))
try:
yield from parser(stream)
except OSError as err:
log.error("Failed to extract {0} streams: {1}".format(parser_name, err))
def _get_streams(self):
if self.is_live:
self.logger.debug("Loading live stream for {0}...", self.channel)
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
self.logger.debug("Attempting to find VOD stream...", self.channel)
vod_data = self.vod_data()
if vod_data:
self.logger.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def test_update_scheme(self):
self.assertEqual(
"https://example.com/foo", # becomes https
update_scheme("https://other.com/bar", "//example.com/foo")
)
self.assertEqual(
"http://example.com/foo", # becomes http
update_scheme("http://other.com/bar", "//example.com/foo")
)
self.assertEqual(
"http://example.com/foo", # remains unchanged
update_scheme("https://other.com/bar", "http://example.com/foo")
)
self.assertEqual(
"https://example.com/foo", # becomes https
update_scheme("https://other.com/bar", "example.com/foo")
)
def _get_streams(self):
data = http.get(self.url, schema=self.config_schema)
for info in data["files"].values():
stream_url = update_scheme(self.url, info["url"])
# pick the smaller of the two dimensions, for landscape v. portrait videos
res = min(info["width"], info["height"])
yield "{0}p".format(res), HTTPStream(self.session, stream_url)
def merge_path_list(self, static, user):
for _path_url in user:
if not _path_url.startswith(('http', '//')):
_path_url = update_scheme('http://', _path_url)
_parsed_path_url = urlparse(_path_url)
if _parsed_path_url.netloc and _parsed_path_url.path:
static += [(_parsed_path_url.netloc, _parsed_path_url.path)]
return static
def live_api_url(self):
channel = self.channel
if channel == "1tv":
url = self._1tv_api
else:
url = self._ctc_api.format(channel=channel)
return update_scheme(self.url, url)
def _get_streams(self):
data = self.session.http.get(self.url, schema=self.config_schema)
for info in data["files"].values():
stream_url = update_scheme(self.url, info["url"])
# pick the smaller of the two dimensions, for landscape v. portrait videos
res = min(info["width"], info["height"])
yield "{0}p".format(res), HTTPStream(self.session, stream_url)
def live_api_url(self):
channel = self.channel
if channel == "1tv":
url = self._1tv_api
else:
url = self._ctc_api.format(channel=channel)
return update_scheme(self.url, url)
def find_iframe(self, url):
res = http.get(url)
# find iframe url
iframe = self.iframe_re.search(res.text)
iframe_url = iframe and iframe.group(1)
if iframe_url:
iframe_url = update_scheme(self.url, iframe_url)
self.logger.debug("Found iframe: {}", iframe_url)
return iframe_url
def _get_streams(self):
res = http.get(self.url)
mobile_url_m = self.mobile_url_re.search(res.text)
desktop_url_m = self.desktop_url_re.search(res.text)
desktop_url = desktop_url_m and update_scheme(
self.url, desktop_url_m.group("url"))
mobile_url = mobile_url_m and update_scheme(self.url,
mobile_url_m.group("url"))
token = (desktop_url_m and desktop_url_m.group("token")) or (
mobile_url_m and mobile_url_m.group("token"))
if not token:
# if no token is in the url, try to find it else where in the page
token_m = self.token_re.search(res.text)
token = token_m and token_m.group("token")
return self._get_star_streams(desktop_url, mobile_url, token=token)
def _get_http_streams(self, info):
name = QUALITY_MAP.get(info["_quality"], "vod")
urls = info["_stream"]
if not isinstance(info["_stream"], list):
urls = [urls]
for url in urls:
stream = HTTPStream(self.session, update_scheme("https://", url))
yield name, stream
def _get_streams(self):
match = _url_re.match(self.url)
channel = match.group("channel")
http.headers.update({"User-Agent": useragents.IPAD})
# Some problem with SSL on huya.com now, do not use https
hls_url = http.get(HUYA_URL % channel, schema=_hls_schema)
yield "live", HLSStream(self.session, update_scheme("http://", hls_url))
def find_iframe(self, url):
res = http.get(url)
# find iframe url
iframe = self.iframe_re.search(res.text)
iframe_url = iframe and iframe.group(1)
if iframe_url:
iframe_url = update_scheme(self.url, iframe_url)
self.logger.debug("Found iframe: {}", iframe_url)
return iframe_url
def _get_streams(self):
page = self.session.http.get(self.url)
for iframe in itertags(page.text, u"iframe"):
url = iframe.attributes["src"]
log.debug("Handing off of {0}".format(url))
try:
return self.session.streams(update_scheme(self.url, url))
except NoPluginError:
log.error("Handing off of {0} failed".format(url))
return None
def _get_streams(self):
page = self.session.http.get(self.url)
for iframe in itertags(page.text, u"iframe"):
url = iframe.attributes["src"]
self.logger.debug("Handing off of {0}".format(url))
try:
return self.session.streams(update_scheme(self.url, url))
except NoPluginError:
self.logger.error("Handing off of {0} failed".format(url))
return None
def _get_streams(self):
page = self.session.http.get(self.url)
iframe = self.iframe_re.search(page.text)
if iframe:
self.logger.debug("Handing off of {0}".format(iframe.group("url")))
try:
return self.session.streams(update_scheme(self.url, iframe.group("url")))
except NoPluginError:
self.logger.error("Handing off of {0} failed".format(iframe.group("url")))
return None
def _get_streams(self):
match = _url_re.match(self.url)
channel = match.group("channel")
self.session.http.headers.update({"User-Agent": useragents.IPAD})
# Some problem with SSL on huya.com now, do not use https
hls_url = self.session.http.get(HUYA_URL % channel, schema=_hls_schema)
yield "live", HLSStream(self.session,
update_scheme("http://", hls_url))
def _get_streams(self):
page = http.get(self.url)
iframe = self.iframe_re.search(page.text)
if iframe:
self.logger.debug("Handing off of {0}".format(iframe.group("url")))
try:
return self.session.streams(update_scheme(self.url, iframe.group("url")))
except NoPluginError:
self.logger.error("Handing off of {0} failed".format(iframe.group("url")))
return None
def _get_streams(self):
res = http.get(self.url)
m = self.embed_url_re.search(res.text)
platform_url = m and m.group("url")
if platform_url:
url = update_scheme(self.url, platform_url)
# hand off to ThePlatform plugin
p = ThePlatform(url)
p.bind(self.session, "plugin.nbcsports")
return p.streams()
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme("http://", urlnoproto)
self.logger.debug("URL={0}; params={1}", urlnoproto, params)
streams = HLSStream.parse_variant_playlist(self.session, urlnoproto, **params)
if not streams:
return {"live": HLSStream(self.session, urlnoproto, **params)}
else:
return streams
def _get_streams(self):
res = self.session.http.get(self.url)
for iframe in itertags(res.text, "iframe"):
self.logger.debug("Found iframe: {0}".format(iframe))
iframe_res = self.session.http.get(iframe.attributes['src'], headers={"Referer": self.url})
m = self.src_re.search(iframe_res.text)
surl = m and m.group("url")
if surl:
surl = update_scheme(self.url, surl)
self.logger.debug("Found stream URL: {0}".format(surl))
return HLSStream.parse_variant_playlist(self.session, surl)
def get_iframe_url(self):
self.logger.debug('search for an iframe')
res = http.get(self.url)
m = self._iframe_re.search(res.text)
if not m:
raise PluginError('No iframe found.')
iframe_url = m.group('url')
iframe_url = update_scheme('http://', iframe_url)
self.logger.debug('IFRAME URL={0}'.format(iframe_url))
return iframe_url
def _get_streams(self):
res = http.get(self.url)
# Search for the iframe in the page
iframe_m = self.iframe_re.search(res.text)
ustream_url = iframe_m and iframe_m.group(1)
if ustream_url and "ustream.tv" in ustream_url:
try:
ustream_url = update_scheme(self.url, ustream_url)
return self.session.streams(ustream_url)
except NoPluginError:
raise PluginError("Could not play embedded stream: {0}".format(ustream_url))
def _get_streams(self):
"""
Find the streams for vk.com
:return:
"""
self.session.http.headers.update({'User-Agent': useragents.IPHONE_6})
# If this is a 'videos' catalog URL
# with an video ID in the GET request, get that instead
url = self.follow_vk_redirect(self.url)
m = self._url_re.match(url)
if not m:
log.error('URL is not compatible: {0}'.format(url))
return
video_id = m.group('video_id')
log.debug('video ID: {0}'.format(video_id))
params = {
'act': 'show_inline',
'al': '1',
'video': video_id,
}
res = self.session.http.post(self.API_URL, params=params)
for _i in itertags(res.text, 'iframe'):
if _i.attributes.get('src'):
iframe_url = update_scheme(self.url, _i.attributes['src'])
log.debug('Found iframe: {0}'.format(iframe_url))
for s in self.session.streams(iframe_url).items():
yield s
for _i in itertags(res.text, 'source'):
if _i.attributes.get('type') == 'application/vnd.apple.mpegurl':
video_url = _i.attributes['src']
# Remove invalid URL
if video_url.startswith('https://vk.com/'):
continue
streams = HLSStream.parse_variant_playlist(self.session,
video_url)
if not streams:
yield 'live', HLSStream(self.session, video_url)
else:
for s in streams.items():
yield s
elif _i.attributes.get('type') == 'video/mp4':
q = 'vod'
video_url = _i.attributes['src']
m = self._vod_quality_re.search(video_url)
if m:
q = '{0}p'.format(m.group(1))
yield q, HTTPStream(self.session, video_url)
def _get_streams(self):
http.headers = {"User-Agent": useragents.CHROME}
res = http.get(self.url)
iframe_url = self.find_iframe(res)
if iframe_url:
self.logger.debug("Found iframe: {0}", iframe_url)
res = http.get(iframe_url, headers={"Referer": self.url})
stream_url = update_scheme(self.url, self.stream_schema.validate(res.text))
return HLSStream.parse_variant_playlist(self.session,
stream_url,
headers={"User-Agent": useragents.CHROME})
def _get_streams(self):
self.session.http.headers.update({"User-Agent": useragents.CHROME})
res = self.session.http.get(self.url)
iframe_url = self.find_iframe(res)
if iframe_url:
log.debug("Found iframe: {0}", iframe_url)
res = self.session.http.get(iframe_url, headers={"Referer": self.url})
stream_url = update_scheme(self.url, self.stream_schema.validate(res.text))
log.warning("SSL Verification disabled.")
return HLSStream.parse_variant_playlist(self.session,
stream_url,
verify=False)
def _get_streams(self):
self.session.set_option('hls-live-edge', 10)
res = self.session.http.get(self.url)
playlist_m = self._playlist_re.search(res.text)
if playlist_m:
return HLSStream.parse_variant_playlist(
self.session,
update_scheme(self.url, playlist_m.group(1)),
headers={'Referer': self.url, 'User-Agent': useragents.ANDROID}
)
else:
log.debug("Could not find stream data")
def _get_streams(self):
res = self.session.http.get(self.url)
mobile_url_m = self.mobile_url_re.search(res.text)
mobile_url = mobile_url_m and update_scheme(self.url, mobile_url_m.group("url"))
token = mobile_url_m and mobile_url_m.group("token")
if not token:
# if no token is in the url, try to find it else where in the page
token_m = self.token_re.search(res.text)
token = token_m and token_m.group("token")
return HLSStream.parse_variant_playlist(self.session, mobile_url + token,
headers={"Referer": self.url})
def _get_streams(self):
http.headers.update({'User-Agent': useragents.CHROME,
'Referer': 'http://www.abweb.com/BIS-TV-Online/bistvo-tele-universal.aspx'})
login_username = self.get_option('username')
login_password = self.get_option('password')
if self.options.get('purge_credentials'):
self._session_attributes.set('ASP.NET_SessionId', None, expires=0)
self._session_attributes.set('.abportail1', None, expires=0)
self._authed = False
self.logger.info('All credentials were successfully removed.')
if not self._authed and not (login_username and login_password):
self.logger.error('A login for ABweb is required, use --abweb-username USERNAME --abweb-password PASSWORD')
return
if self._authed:
if self._expires < time.time():
self.logger.debug('get new cached cookies')
# login after 24h
self.set_expires_time_cache()
self._authed = False
else:
self.logger.info('Attempting to authenticate using cached cookies')
http.cookies.set('ASP.NET_SessionId', self._session_attributes.get('ASP.NET_SessionId'))
http.cookies.set('.abportail1', self._session_attributes.get('.abportail1'))
if not self._authed and not self._login(login_username, login_password):
return
iframe_url = self.get_iframe_url()
http.headers.update({'Referer': iframe_url})
hls_url = self.get_hls_url(iframe_url)
hls_url = update_scheme(self.url, hls_url)
self.logger.debug('URL={0}'.format(hls_url))
variant = HLSStream.parse_variant_playlist(self.session, hls_url)
if variant:
for q, s in variant.items():
yield q, s
else:
yield 'live', HLSStream(self.session, hls_url)
def _get_streams_api(self, video_id):
res = self.session.http.get(self.api_server,
params=dict(video_id=video_id))
data = self.session.http.json(res)
if data["success"]:
for x in itertools.chain(*data['data']['versions'].values()):
src = update_scheme(self.url, x['src'])
if x['type'] == "application/x-mpegurl":
for s in HLSStream.parse_variant_playlist(self.session, src).items():
yield s
elif x['type'] == "application/dash+xml":
for s in DASHStream.parse_manifest(self.session, src).items():
yield s
elif x['type'] == "video/mp4":
yield "{0}p".format(x['res']), HTTPStream(self.session, src)
else:
log.error("Failed to get streams: {0} ({1})".format(
data['message'], data['code']
))
def _get_streams(self):
vid = self.url_re.match(self.url).group(1)
self.logger.debug("Found video ID: {0}", vid)
page = self.session.http.get(self.play_url.format(vid=vid))
js_url_m = self.js_re.search(page.text)
if js_url_m:
js_url = js_url_m.group(1)
self.logger.debug("Loading player JS: {0}", js_url)
res = self.session.http.get(js_url)
metadata_url = update_scheme(self.url, self.setup_schema.validate(res.text))
data = self.session.http.json(self.session.http.get(metadata_url))
for source in data["playlist"][0]["sources"]:
if source["type"] == "application/vnd.apple.mpegurl":
for s in HLSStream.parse_variant_playlist(self.session, source["file"]).items():
yield s
elif source["type"] == "video/mp4":
yield "{0}p".format(source["height"]), HTTPStream(self.session, source["file"])
def _get_live_streams(self):
# Get channel id
match = self._url_re.match(self.url)
channel = match.group('channel')
# Retrieve live player URL
res = http.get(self.PLAYER_URL)
match = self._live_player_re.search(res.text)
if match is None:
return []
live_player_url = update_scheme(self.url, match.group('live_player_url'))
# Extract streams from the live player page
res = http.get(live_player_url)
stream_datas = re.findall(r'{0}(?:_MINI)?:({{.+?}}]}}]}})'.format(self.CHANNEL_MAP[channel]), res.text)
streams = []
for s in stream_datas:
for u in self._live_streams_schema.validate(s):
if u not in streams:
streams.append(u)
return streams
def resolve_url(self, url, follow_redirect=True):
"""Attempts to find a plugin that can use this URL.
The default protocol (http) will be prefixed to the URL if
not specified.
Raises :exc:`NoPluginError` on failure.
:param url: a URL to match against loaded plugins
:param follow_redirect: follow redirects
"""
url = update_scheme("http://", url)
available_plugins = []
for name, plugin in self.plugins.items():
if plugin.can_handle_url(url):
available_plugins.append(plugin)
available_plugins.sort(key=lambda x: x.priority(url), reverse=True)
if available_plugins:
return available_plugins[0](url)
if follow_redirect:
# Attempt to handle a redirect URL
try:
res = self.http.head(url, allow_redirects=True, acceptable_status=[501])
# Fall back to GET request if server doesn't handle HEAD.
if res.status_code == 501:
res = self.http.get(url, stream=True)
if res.url != url:
return self.resolve_url(res.url, follow_redirect=follow_redirect)
except PluginError:
pass
raise NoPluginError
def _get_streams(self):
match = url_re.match(self.url)
stream_page_scheme = 'https'
stream_page_domain = match.group(4)
stream_page_path = match.group(5)
country_code = CONST_DEFAULT_COUNTRY_CODE
# create http session and set headers
http_session = http
http_session.headers.update(CONST_HEADERS)
# get cookies
r = http_session.get(urlunparse((stream_page_scheme, stream_page_domain, stream_page_path, '', '', '')))
# redirect to profile page means stream is offline
if '/profile/' in r.url:
raise NoStreamsError(self.url)
if not r.ok:
self.logger.debug("Status code for {0}: {1}", r.url, r.status_code)
raise NoStreamsError(self.url)
if len(http_session.cookies) == 0:
raise PluginError("Can't get a cookies")
if urlparse(r.url).netloc != stream_page_domain:
# then redirected to regional subdomain
country_code = urlparse(r.url).netloc.split('.')[0].lower()
# time to set variables
baseurl = urlunparse((stream_page_scheme, urlparse(r.url).netloc, '', '', '', ''))
amf_gateway_url = urljoin(baseurl, CONST_AMF_GATEWAY_LOCATION)
stream_page_url = urljoin(baseurl, stream_page_path)
headers = {
'User-Agent': useragents.CHROME,
'Referer': stream_page_url,
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Requested-With': 'XMLHttpRequest'
}
data = 'method=getRoomData&args%5B%5D={0}&args%5B%5D=false'.format(stream_page_path)
self.logger.debug('DATA: {0}'.format(str(data)))
# send request and close http-session
r = http_session.post(url=amf_gateway_url,
headers=headers,
params={CONST_AMF_GATEWAY_PARAM: country_code},
data=data)
http_session.close()
if r.status_code != 200:
raise PluginError("unexpected status code for {0}: {1}", r.url, r.status_code)
stream_source_info = amf_msg_schema.validate(json.loads(r.text))
self.logger.debug("source stream info:\n{0}", stream_source_info)
if not stream_source_info:
return
urlnoproto = stream_source_info['localData']['videoServerUrl']
urlnoproto = update_scheme('https://', urlnoproto)
performer = stream_source_info['performerData']['username']
hls_url = '{0}/hls/stream_{1}/playlist.m3u8'.format(urlnoproto, performer)
if hls_url:
self.logger.debug('HLS URL: {0}'.format(hls_url))
try:
for s in HLSStream.parse_variant_playlist(self.session, hls_url, headers=headers).items():
yield s
except Exception as e:
if '404' in str(e):
self.logger.error('Stream is currently offline or private')
else:
self.logger.error(str(e))
return
def set_option(self, key, value):
"""Sets general options used by plugins and streams originating
from this session object.
:param key: key of the option
:param value: value to set the option to
**Available options**:
======================== =========================================
hds-live-edge ( float) Specify the time live HDS
streams will start from the edge of
stream, default: ``10.0``
hds-segment-attempts (int) How many attempts should be done
to download each HDS segment, default: ``3``
hds-segment-threads (int) The size of the thread pool used
to download segments, default: ``1``
hds-segment-timeout (float) HDS segment connect and read
timeout, default: ``10.0``
hds-timeout (float) Timeout for reading data from
HDS streams, default: ``60.0``
hls-live-edge (int) How many segments from the end
to start live streams on, default: ``3``
hls-segment-attempts (int) How many attempts should be done
to download each HLS segment, default: ``3``
hls-segment-threads (int) The size of the thread pool used
to download segments, default: ``1``
hls-segment-timeout (float) HLS segment connect and read
timeout, default: ``10.0``
hls-timeout (float) Timeout for reading data from
HLS streams, default: ``60.0``
http-proxy (str) Specify a HTTP proxy to use for
all HTTP requests
https-proxy (str) Specify a HTTPS proxy to use for
all HTTPS requests
http-cookies (dict or str) A dict or a semi-colon (;)
delimited str of cookies to add to each
HTTP request, e.g. ``foo=bar;baz=qux``
http-headers (dict or str) A dict or semi-colon (;)
delimited str of headers to add to each
HTTP request, e.g. ``foo=bar;baz=qux``
http-query-params (dict or str) A dict or a ampersand (&)
delimited string of query parameters to
add to each HTTP request,
e.g. ``foo=bar&baz=qux``
http-trust-env (bool) Trust HTTP settings set in the
environment, such as environment
variables (HTTP_PROXY, etc) and
~/.netrc authentication
http-ssl-verify (bool) Verify SSL certificates,
default: ``True``
http-ssl-cert (str or tuple) SSL certificate to use,
can be either a .pem file (str) or a
.crt/.key pair (tuple)
http-timeout (float) General timeout used by all HTTP
requests except the ones covered by
other options, default: ``20.0``
http-stream-timeout (float) Timeout for reading data from
HTTP streams, default: ``60.0``
subprocess-errorlog (bool) Log errors from subprocesses to
a file located in the temp directory
subprocess-errorlog-path (str) Log errors from subprocesses to
a specific file
ringbuffer-size (int) The size of the internal ring
buffer used by most stream types,
default: ``16777216`` (16MB)
rtmp-proxy (str) Specify a proxy (SOCKS) that RTMP
streams will use
rtmp-rtmpdump (str) Specify the location of the
rtmpdump executable used by RTMP streams,
e.g. ``/usr/local/bin/rtmpdump``
rtmp-timeout (float) Timeout for reading data from
RTMP streams, default: ``60.0``
ffmpeg-ffmpeg (str) Specify the location of the
ffmpeg executable use by Muxing streams
e.g. ``/usr/local/bin/ffmpeg``
ffmpeg-verbose (bool) Log stderr from ffmpeg to the
console
ffmpeg-verbose-path (str) Specify the location of the
ffmpeg stderr log file
ffmpeg-video-transcode (str) The codec to use if transcoding
video when muxing with ffmpeg
e.g. ``h264``
ffmpeg-audio-transcode (str) The codec to use if transcoding
audio when muxing with ffmpeg
e.g. ``aac``
stream-segment-attempts (int) How many attempts should be done
to download each segment, default: ``3``.
General option used by streams not
covered by other options.
stream-segment-threads (int) The size of the thread pool used
to download segments, default: ``1``.
General option used by streams not
covered by other options.
stream-segment-timeout (float) Segment connect and read
timeout, default: ``10.0``.
General option used by streams not
covered by other options.
stream-timeout (float) Timeout for reading data from
stream, default: ``60.0``.
General option used by streams not
covered by other options.
locale (str) Locale setting, in the RFC 1766 format
eg. en_US or es_ES
default: ``system locale``.
======================== =========================================
"""
# Backwards compatibility
if key == "rtmpdump":
key = "rtmp-rtmpdump"
elif key == "rtmpdump-proxy":
key = "rtmp-proxy"
elif key == "errorlog":
key = "subprocess-errorlog"
elif key == "errorlog-path":
key = "subprocess-errorlog-path"
if key == "http-proxy":
self.http.proxies["http"] = update_scheme("http://", value)
elif key == "https-proxy":
self.http.proxies["https"] = update_scheme("https://", value)
elif key == "http-cookies":
if isinstance(value, dict):
self.http.cookies.update(value)
else:
self.http.parse_cookies(value)
elif key == "http-headers":
if isinstance(value, dict):
self.http.headers.update(value)
else:
self.http.parse_headers(value)
elif key == "http-query-params":
if isinstance(value, dict):
self.http.params.update(value)
else:
self.http.parse_query_params(value)
elif key == "http-trust-env":
self.http.trust_env = value
elif key == "http-ssl-verify":
self.http.verify = value
elif key == "http-disable-dh":
if value:
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':!DH'
try:
requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST = \
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS.encode("ascii")
except AttributeError:
# no ssl to disable the cipher on
pass
elif key == "http-ssl-cert":
self.http.cert = value
elif key == "http-timeout":
self.http.timeout = value
else:
self.options.set(key, value)
