def _parse_streams(self, res):
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def test_stream_open_video_only(self, muxer, reader):
stream = DASHStream(self.session, Mock(), Mock(id=1, mimeType="video/mp4"))
open_reader = reader.return_value = Mock()
stream.open()
reader.assert_called_with(stream, 1, "video/mp4")
open_reader.open.assert_called_with()
muxer.assert_not_called()
def test_stream_open_video_audio(self, muxer, reader):
stream = DASHStream(self.session, Mock(), Mock(id=1, mimeType="video/mp4"), Mock(id=2, mimeType="audio/mp3", lang='en'))
open_reader = reader.return_value = Mock()
stream.open()
self.assertSequenceEqual(reader.mock_calls, [call(stream, 1, "video/mp4"),
call().open(),
call(stream, 2, "audio/mp3"),
call().open()])
self.assertSequenceEqual(muxer.mock_calls, [call(self.session, open_reader, open_reader, copyts=True),
call().open()])
def _parse_streams(self, res):
_found_stream_url = False
for meta in itertags(res.text, "meta"):
if meta.attributes.get("property") == "og:video:url":
stream_url = html_unescape(meta.attributes.get("content"))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
_found_stream_url = True
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
_found_stream_url = True
break
else:
log.debug("No meta og:video:url")
if _found_stream_url:
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def _get_streams(self):
if self.is_live:
log.debug("Loading live stream for {0}...".format(self.channel))
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
log.debug("Attempting to find VOD stream for {0}...".format(self.channel))
vod_data = self.vod_data()
if vod_data:
log.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def test_parse_manifest_audio_multi(self, mpdClass):
mpdClass.return_value = Mock(periods=[
Mock(adaptationSets=[
Mock(contentProtection=None,
representations=[
Mock(id=1, mimeType="video/mp4", height=720),
Mock(id=2, mimeType="video/mp4", height=1080),
Mock(id=3,
mimeType="audio/aac",
bandwidth=128.0,
lang='en'),
Mock(id=4,
mimeType="audio/aac",
bandwidth=256.0,
lang='en')
])
])
])
streams = DASHStream.parse_manifest(self.session, self.test_url)
mpdClass.assert_called_with(ANY,
base_url="http://test.bar",
url="http://test.bar/foo.mpd")
self.assertSequenceEqual(
sorted(list(streams.keys())),
sorted(["720p+a128k", "1080p+a128k", "720p+a256k", "1080p+a256k"]))
def _get_streams(self):
m = self.match
if m:
channel = m.group(1) or m.group(2)
log.debug("Found channel {0}".format(channel))
for sformat, url in self.get_stream_urls(channel):
try:
if sformat == "dash":
yield from DASHStream.parse_manifest(
self.session,
url,
headers={
"User-Agent": useragents.CHROME
}).items()
if sformat == "hls":
yield from HLSStream.parse_variant_playlist(
self.session,
url,
headers={
"User-Agent": useragents.IPHONE
},
).items()
except PluginError as e:
log.error("Could not open {0} stream".format(sformat))
log.debug("Failed with error: {0}".format(e))
def _get_streams(self):
res = self.session.http.get(self.url, headers={"User-Agent": useragents.CHROME})
streams = {}
vod_urls = set([])
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
streams.update(DASHStream.parse_manifest(self.session, stream_url))
elif ".mp4" in stream_url:
streams[match.group(1)] = HTTPStream(self.session, stream_url)
vod_urls.add(stream_url)
else:
self.logger.debug("Non-dash/mp4 stream: {0}".format(stream_url))
if streams:
return streams
# fallback on to playlist
self.logger.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
for url in dict(url.group(1) for url in self._plurl_re.finditer(playlist)):
if url not in vod_urls:
streams["sd"] = HTTPStream(self.session, url)
return streams
def _get_streams(self):
res = http.get(self.url, headers={"User-Agent": useragents.CHROME})
streams = {}
vod_urls = set([])
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
streams.update(DASHStream.parse_manifest(self.session, stream_url))
elif ".mp4" in stream_url:
streams[match.group(1)] = HTTPStream(self.session, stream_url)
vod_urls.add(stream_url)
else:
self.logger.debug("Non-dash/mp4 stream: {0}".format(stream_url))
if streams:
return streams
# fallback on to playlist
self.logger.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
# for url in {url.group(1) for url in self._plurl_re.finditer(playlist)}:
for url in dict(url.group(1) for url in self._plurl_re.finditer(playlist)):
if url not in vod_urls:
streams["sd"] = HTTPStream(self.session, url)
return streams
def _get_streams(self):
api_urls = self.session.http.get(self.url,
schema=self.channel_id_schema)
_api_url = list(api_urls)[0]
log.debug("API URL: {0}".format(_api_url))
player_api_url = self.session.http.get(_api_url,
schema=self.player_api_schema)
for api_url in player_api_url:
log.debug("Player API URL: {0}".format(api_url))
for source in self.session.http.get(api_url,
schema=self.stream_schema):
log.debug("Stream source: {0} ({1})".format(
source['src'], source.get("type", "n/a")))
if "type" not in source or source[
"type"] == "application/vnd.apple.mpegurl":
streams = HLSStream.parse_variant_playlist(
self.session, source["src"])
if not streams:
yield "live", HLSStream(self.session, source["src"])
else:
for s in streams.items():
yield s
elif source["type"] == "application/dash+xml":
for s in DASHStream.parse_manifest(self.session,
source["src"]).items():
yield s
def test_parse_manifest_with_duplicated_resolutions(self, mpdClass):
"""
Verify the fix for https://github.com/streamlink/streamlink/issues/3365
"""
mpdClass.return_value = Mock(periods=[
Mock(adaptationSets=[
Mock(contentProtection=None,
representations=[
Mock(id=1,
mimeType="video/mp4",
height=1080,
bandwidth=128.0),
Mock(id=2,
mimeType="video/mp4",
height=1080,
bandwidth=64.0),
Mock(id=3,
mimeType="video/mp4",
height=1080,
bandwidth=32.0),
Mock(id=4, mimeType="video/mp4", height=720),
])
])
])
streams = DASHStream.parse_manifest(self.session, self.test_url)
mpdClass.assert_called_with(ANY,
base_url="http://test.bar",
url="http://test.bar/foo.mpd")
self.assertSequenceEqual(
sorted(list(streams.keys())),
sorted(["720p", "1080p", "1080p_alt", "1080p_alt2"]))
def _get_video_streams(self):
res = self.session.http.get(self.url)
match = self._video_player_re.search(res.text)
if match is None:
return
player_url = match.group('player_url')
stream_data = self.session.http.get(player_url, schema=self._video_stream_schema)
if stream_data is None:
return
# Check geolocation to prevent further errors when stream is parsed
if not self.check_geolocation(stream_data['geoLocRestriction']):
self.logger.error('Stream is geo-restricted')
return
# Check whether streams are DRM-protected
if stream_data.get('drm', False):
self.logger.error('Stream is DRM-protected')
return
now = datetime.datetime.now()
try:
if isinstance(stream_data['sources'], dict):
urls = []
for profile, url in stream_data['sources'].items():
if not url or url in urls:
continue
match = self._stream_size_re.match(url)
if match is not None:
quality = match.group('size')
else:
quality = profile
yield quality, HTTPStream(self.session, url)
urls.append(url)
hls_url = stream_data.get('urlHls') or stream_data.get('streamUrlHls')
if hls_url:
if stream_data.get('isLive', False):
# Live streams require a token
hls_url = self.tokenize_stream(hls_url)
for stream in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield stream
dash_url = stream_data.get('urlDash') or stream_data.get('streamUrlDash')
if dash_url:
if stream_data.get('isLive', False):
# Live streams require a token
dash_url = self.tokenize_stream(dash_url)
for stream in DASHStream.parse_manifest(self.session, dash_url).items():
yield stream
except IOError as err:
if '403 Client Error' in str(err):
# Check whether video is expired
if 'startDate' in stream_data:
if now < self.iso8601_to_epoch(stream_data['startDate']):
self.logger.error('Stream is not yet available')
elif 'endDate' in stream_data:
if now > self.iso8601_to_epoch(stream_data['endDate']):
self.logger.error('Stream has expired')
def _get_streams(self):
if self.is_live:
self.logger.debug("Loading live stream for {0}...", self.channel)
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
self.logger.debug("Attempting to find VOD stream...", self.channel)
vod_data = self.vod_data()
if vod_data:
self.logger.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_video_streams(self):
res = http.get(self.url)
match = self._video_player_re.search(res.text)
if match is None:
return
player_url = match.group('player_url')
stream_data = http.get(player_url, schema=self._video_stream_schema)
if stream_data is None:
return
# Check geolocation to prevent further errors when stream is parsed
if not self.check_geolocation(stream_data['geoLocRestriction']):
self.logger.error('Stream is geo-restricted')
return
# Check whether streams are DRM-protected
if stream_data.get('drm', False):
self.logger.error('Stream is DRM-protected')
return
now = datetime.datetime.now()
try:
if isinstance(stream_data['sources'], dict):
urls = []
for profile, url in stream_data['sources'].items():
if not url or url in urls:
continue
match = self._stream_size_re.match(url)
if match is not None:
quality = match.group('size')
else:
quality = profile
yield quality, HTTPStream(self.session, url)
urls.append(url)
hls_url = stream_data.get('urlHls') or stream_data.get('streamUrlHls')
if hls_url:
if stream_data.get('isLive', False):
# Live streams require a token
hls_url = self.tokenize_stream(hls_url)
for stream in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield stream
dash_url = stream_data.get('urlDash') or stream_data.get('streamUrlDash')
if dash_url:
if stream_data.get('isLive', False):
# Live streams require a token
dash_url = self.tokenize_stream(dash_url)
for stream in DASHStream.parse_manifest(self.session, dash_url).items():
yield stream
except IOError as err:
if '403 Client Error' in str(err):
# Check whether video is expired
if 'startDate' in stream_data:
if now < self.iso8601_to_epoch(stream_data['startDate']):
self.logger.error('Stream is not yet available')
elif 'endDate' in stream_data:
if now > self.iso8601_to_epoch(stream_data['endDate']):
self.logger.error('Stream has expired')
def test_parse_manifest_audio_multi_lang_locale(self, mpdClass):
self.session.localization.language.alpha2 = "es"
self.session.localization.explicit = True
mpdClass.return_value = Mock(periods=[
Mock(adaptationSets=[
Mock(contentProtection=None,
representations=[
Mock(id=1, mimeType="video/mp4", height=720),
Mock(id=2, mimeType="video/mp4", height=1080),
Mock(id=3,
mimeType="audio/aac",
bandwidth=128.0,
lang='en'),
Mock(id=4,
mimeType="audio/aac",
bandwidth=128.0,
lang='es')
])
])
])
streams = DASHStream.parse_manifest(self.session, self.test_url)
mpdClass.assert_called_with(ANY,
base_url="http://test.bar",
url="http://test.bar/foo.mpd")
self.assertSequenceEqual(sorted(list(streams.keys())),
sorted(["720p", "1080p"]))
self.assertEqual(streams["720p"].audio_representation.lang, "es")
self.assertEqual(streams["1080p"].audio_representation.lang, "es")
def test_segments_number_time(self, mpdClass):
with xml("dash/test_9.mpd") as mpd_xml:
mpdClass.return_value = MPD(mpd_xml, base_url="http://test.bar", url="http://test.bar/foo.mpd")
streams = DASHStream.parse_manifest(self.session, self.test_url)
mpdClass.assert_called_with(ANY, base_url="http://test.bar", url="http://test.bar/foo.mpd")
self.assertSequenceEqual(list(streams.keys()), ['2500k'])
def _watch(self):
log.debug('_watch ...')
channel = self.match.group('channel')
vod_id = self.match.group('vod_id')
recording_id = self.match.group('recording_id')
params = {'https_watch_urls': True}
if channel:
watch_url = f'{self.base_url}/zapi/watch'
params_cid = self._get_params_cid(channel)
if not params_cid:
return
params.update(params_cid)
elif vod_id:
log.debug('Found vod_id: {0}'.format(vod_id))
watch_url = f'{self.base_url}/zapi/avod/videos/{vod_id}/watch'
elif recording_id:
log.debug('Found recording_id: {0}'.format(recording_id))
watch_url = f'{self.base_url}/zapi/watch/recording/{recording_id}'
else:
log.debug('Missing watch_url')
return
zattoo_stream_types = self.get_option('stream-types')
for stream_type in zattoo_stream_types:
params_stream_type = {'stream_type': stream_type}
params.update(params_stream_type)
try:
res = self.session.http.post(watch_url,
headers=self.headers,
data=params)
except Exception as e:
if '404 Client Error' in str(e):
log.error('Unfortunately streaming is not permitted in '
'this country or this channel does not exist.')
elif '402 Client Error: Payment Required' in str(e):
log.error('Paid subscription required for this channel.')
log.info('If paid subscription exist, use --zattoo-purge'
'-credentials to start a new session.')
elif '403 Client Error' in str(e):
log.debug('Force session reset for watch_url')
self.reset_session()
else:
log.error(str(e))
return
data = self.session.http.json(res)
log.debug('Found data for {0}'.format(stream_type))
if data['success'] and stream_type == 'hls5':
for url in data['stream']['watch_urls']:
yield from HLSStream.parse_variant_playlist(
self.session, url['url']).items()
elif data['success'] and stream_type == 'dash':
for url in data['stream']['watch_urls']:
yield from DASHStream.parse_manifest(
self.session, url['url']).items()
def _get_streams(self):
if "player.vimeo.com" in self.url:
data = self.session.http.get(self.url, schema=self._player_schema)
else:
api_url = self.session.http.get(self.url,
schema=self._config_url_schema)
if not api_url:
return
data = self.session.http.get(api_url, schema=self._config_schema)
videos = data["request"]["files"]
streams = []
for stream_type in ("hls", "dash"):
if stream_type not in videos:
continue
for _, video_data in videos[stream_type]["cdns"].items():
log.trace("{0!r}".format(video_data))
url = video_data.get("url")
if stream_type == "hls":
for stream in HLSStream.parse_variant_playlist(
self.session, url).items():
streams.append(stream)
elif stream_type == "dash":
p = urlparse(url)
if p.path.endswith("dash.mpd"):
# LIVE
url = self.session.http.get(url).json()["url"]
elif p.path.endswith("master.json"):
# VOD
url = url.replace("master.json", "master.mpd")
else:
log.error("Unsupported DASH path: {0}".format(p.path))
continue
for stream in DASHStream.parse_manifest(self.session,
url).items():
streams.append(stream)
for stream in videos.get("progressive", []):
streams.append(
(stream["quality"], HTTPStream(self.session, stream["url"])))
if self.get_option("mux_subtitles") and data["request"].get(
"text_tracks"):
substreams = {
s["lang"]: HTTPStream(self.session,
"https://vimeo.com" + s["url"])
for s in data["request"]["text_tracks"]
}
for quality, stream in streams:
yield quality, MuxedStream(self.session,
stream,
subtitles=substreams)
else:
for stream in streams:
yield stream
def _get_streams(self):
api_urls = self.session.http.get(self.url, schema=self.channel_id_schema)
for api_url in api_urls:
log.debug("API URL: {0}".format(api_url))
for source in self.session.http.get(api_url, schema=self.stream_schema):
log.debug("Stream source: {0} ({1})".format(source['src'], source.get("type", "n/a")))
if "type" not in source or source["type"] == "application/vnd.apple.mpegurl":
for s in HLSStream.parse_variant_playlist(self.session, source["src"]).items():
yield s
elif source["type"] == "application/dash+xml":
for s in DASHStream.parse_manifest(self.session, source["src"]).items():
yield s
def _get_streams(self):
api_urls = http.get(self.url, schema=self.channel_id_schema)
for api_url in api_urls:
log.debug("API URL: {0}".format(api_url))
for source in http.get(api_url, schema=self.stream_schema):
log.debug("Stream source: {0} ({1})".format(
source['src'], source.get("type", "n/a")))
if "type" not in source or source[
"type"] == "application/vnd.apple.mpegurl":
for s in HLSStream.parse_variant_playlist(
self.session, source["src"]).items():
yield s
elif source["type"] == "application/dash+xml":
for s in DASHStream.parse_manifest(self.session,
source["src"]).items():
yield s
def _get_streams(self):
m = self.url_re.match(self.url)
if m:
channel = m.group(1) or m.group(2)
self.logger.debug("Found channel {0}", channel)
data = self.api_call(channel)
if data.get("error"):
log.error(
"Failed to get stream for {0}: {error} ({code})".format(
channel, **data))
else:
log.debug("Got {format} stream {url}".format(**data))
if data["format"] == "dash":
for s in DASHStream.parse_manifest(self.session,
data["url"]).items():
yield s
def _get_live(self, path):
match = self.live_id_re.search(path)
if match is None:
return
live_id = "ch-{0}".format(match.group('live_id'))
log.debug("Live ID={0}".format(live_id))
res = self.session.http.get(self.api_url.format(live_id))
api_data = self.session.http.json(res, schema=self._video_schema)
self._set_metadata(api_data, 'Live')
for playlist in api_data['videoReferences']:
if playlist['format'] == 'dashhbbtv':
for s in DASHStream.parse_manifest(self.session, playlist['url']).items():
yield s
def test_parse_manifest_video_only(self, mpdClass):
mpdClass.return_value = Mock(periods=[
Mock(adaptationSets=[
Mock(contentProtection=None,
representations=[
Mock(id=1, mimeType="video/mp4", height=720),
Mock(id=2, mimeType="video/mp4", height=1080)
])
])
])
streams = DASHStream.parse_manifest(self.session, self.test_url)
mpdClass.assert_called_with(ANY,
base_url="http://test.bar",
url="http://test.bar/foo.mpd")
self.assertSequenceEqual(sorted(list(streams.keys())),
sorted(["720p", "1080p"]))
def _get_streams_api(self, video_id):
res = self.session.http.get(self.api_server,
params=dict(video_id=video_id))
data = self.session.http.json(res)
if data["success"]:
for x in itertools.chain(*data['data']['versions'].values()):
src = update_scheme(self.url, x['src'])
if x['type'] == "application/x-mpegurl":
for s in HLSStream.parse_variant_playlist(self.session, src).items():
yield s
elif x['type'] == "application/dash+xml":
for s in DASHStream.parse_manifest(self.session, src).items():
yield s
elif x['type'] == "video/mp4":
yield "{0}p".format(x['res']), HTTPStream(self.session, src)
else:
log.error("Failed to get streams: {0} ({1})".format(
data['message'], data['code']
))
def _get_streams_api(self, video_id):
res = self.session.http.get(self.api_server,
params=dict(video_id=video_id))
data = self.session.http.json(res)
if data["success"]:
for x in itertools.chain(*data['data']['versions'].values()):
src = update_scheme(self.url, x['src'])
if x['type'] == "application/x-mpegurl":
yield from HLSStream.parse_variant_playlist(
self.session, src).items()
elif x['type'] == "application/dash+xml":
yield from DASHStream.parse_manifest(self.session,
src).items()
elif x['type'] == "video/mp4":
yield "{0}p".format(x['res']), HTTPStream(
self.session, src)
else:
log.error("Failed to get streams: {0} ({1})".format(
data['message'], data['code']))
def _get_streams(self):
page = self.session.http.get(self.url)
api_info = self._get_api_info(page)
if not api_info:
log.error("Could not find API info in page")
return
token_res = self.session.http.post(api_info["token_url"])
token = self.session.http.json(token_res, schema=self._token_schema)
log.debug("Got token: {0}".format(token))
log.debug("Getting stream data: {0}".format(api_info["stream_url"]))
res = self.session.http.get(api_info["stream_url"],
params={
"vrtPlayerToken": token,
"client": "vrtvideo"
},
raise_for_status=False)
data = self.session.http.json(res, schema=self._stream_schema)
if "code" in data:
log.error("{0} ({1})".format(data['message'], data['code']))
return
log.debug(
"Streams have {0}DRM".format("no " if not data["drm"] else ""))
for target in data["targetUrls"]:
if data["drm"]:
if target["type"] == "hls_aes":
for s in HLSStream.parse_variant_playlist(
self.session, target["url"]).items():
yield s
elif target["type"] == "hls":
for s in HLSStream.parse_variant_playlist(
self.session, target["url"]).items():
yield s
elif target["type"] == "mpeg_dash":
for s in DASHStream.parse_manifest(self.session,
target["url"]).items():
yield s
def _get_vod(self):
res = self.session.http.get(self.url)
match = self.latest_episode_url_re.search(res.text)
if match:
res = self.session.http.get(urljoin(self.url,
match.group('url')), )
match = self.vod_id_re.search(res.text)
if match is None:
return
vod_id = match.group('vod_id')
log.debug("VOD ID={0}".format(vod_id))
res = self.session.http.get(self.api_url.format(vod_id))
api_data = self.session.http.json(res, schema=self._video_schema)
self._set_metadata(api_data, 'VOD')
substreams = {}
if 'subtitleReferences' in api_data:
for subtitle in api_data['subtitleReferences']:
if subtitle['format'] == 'webvtt':
log.debug("Subtitle={0}".format(subtitle['url']))
substreams[subtitle['format']] = HTTPStream(
self.session,
subtitle['url'],
)
for manifest in api_data['videoReferences']:
if manifest['format'] == 'dashhbbtv':
for q, s in DASHStream.parse_manifest(self.session,
manifest['url']).items():
if self.get_option('mux_subtitles') and substreams:
yield q, MuxedStream(self.session,
s,
subtitles=substreams)
else:
yield q, s
def _get_streams(self):
res = http.get(self.url, headers={"User-Agent": useragents.CHROME})
with open("temp.html", "w") as f:
f.write(res.text)
for match in self._mpd_re.finditer(res.text):
manifest_url = match.group("url")
if "\\/" in manifest_url:
# if the URL is json encoded, decode it
manifest_url = parse_json("\"{}\"".format(manifest_url))
for s in DASHStream.parse_manifest(self.session,
manifest_url).items():
yield s
else:
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
for url in {
url.group(1)
for url in self._plurl_re.finditer(playlist)
}:
yield "live", HTTPStream(self.session, url)
def _get_streams(self):
page = self.session.http.get(self.url)
api_info = self._get_api_info(page)
if not api_info:
log.error("Could not find API info in page")
return
token_res = self.session.http.post(api_info["token_url"])
token = self.session.http.json(token_res, schema=self._token_schema)
log.debug("Got token: {0}".format(token))
log.debug("Getting stream data: {0}".format(api_info["stream_url"]))
res = self.session.http.get(api_info["stream_url"],
params={
"vrtPlayerToken": token,
"client": "vrtvideo"
}, raise_for_status=False)
data = self.session.http.json(res, schema=self._stream_schema)
if "code" in data:
log.error("{0} ({1})".format(data['message'], data['code']))
return
log.debug("Streams have {0}DRM".format("no " if not data["drm"] else ""))
for target in data["targetUrls"]:
if data["drm"]:
if target["type"] == "hls_aes":
for s in HLSStream.parse_variant_playlist(self.session, target["url"]).items():
yield s
elif target["type"] == "hls":
for s in HLSStream.parse_variant_playlist(self.session, target["url"]).items():
yield s
elif target["type"] == "mpeg_dash":
for s in DASHStream.parse_manifest(self.session, target["url"]).items():
yield s
def _get_streams(self):
# Retrieve geolocation data
res = self.session.http.get(self.GEO_URL)
geo = self.session.http.json(res, schema=self._geo_schema)
country_code = geo['reponse']['geo_info']['country_code']
log.debug('Country: {0}'.format(country_code))
# Retrieve URL page and search for video ID
res = self.session.http.get(self.url)
if 'france.tv' in self.url:
match = self._pluzz_video_id_re.search(res.text)
elif 'ludo.fr' in self.url or 'zouzous.fr' in self.url:
match = self._jeunesse_video_id_re.search(res.text)
elif 'sport.francetvinfo.fr' in self.url:
match = self._sport_video_id_re.search(res.text)
else:
match = self._embed_video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
log.debug('Video ID: {0}'.format(video_id))
res = self.session.http.get(self.API_URL.format(video_id))
videos = self.session.http.json(res, schema=self._api_schema)
now = time.time()
offline = False
geolocked = False
drm = False
expired = False
streams = []
for video in videos['videos']:
log.trace('{0!r}'.format(video))
video_url = video['url']
# Check whether video format is available
if video['statut'] != 'ONLINE':
offline = offline or True
continue
# Check whether video format is geo-locked
if video['geoblocage'] is not None and country_code not in video[
'geoblocage']:
geolocked = geolocked or True
continue
# Check whether video is DRM-protected
if video['drm']:
drm = drm or True
continue
# Check whether video format is expired
available = False
for interval in video['plages_ouverture']:
available = (interval['debut'] or 0) <= now <= (interval['fin']
or sys.maxsize)
if available:
break
if not available:
expired = expired or True
continue
res = self.session.http.get(self.TOKEN_URL.format(video_url))
video_url = res.text
if '.mpd' in video_url:
# Get redirect video URL
res = self.session.http.get(res.text)
video_url = res.url
for bitrate, stream in DASHStream.parse_manifest(
self.session, video_url).items():
streams.append((bitrate, stream))
elif '.f4m' in video_url:
for bitrate, stream in HDSStream.parse_manifest(
self.session,
video_url,
is_akamai=True,
pvswf=self.SWF_PLAYER_URL).items():
# HDS videos with data in their manifest fragment token
# doesn't seem to be supported by HDSStream. Ignore such
# stream (but HDS stream having only the hdntl parameter in
# their manifest token will be provided)
pvtoken = stream.request_params['params'].get(
'pvtoken', '')
match = self._hds_pv_data_re.search(pvtoken)
if match is None:
streams.append((bitrate, stream))
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(
self.session, video_url).items():
streams.append(stream)
# HBB TV streams are not provided anymore by France Televisions
elif '.mp4' in video_url and '/hbbtv/' not in video_url:
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
# Fallback bitrate (seems all France Televisions MP4 videos
# seem have such bitrate)
bitrate = '1500k'
streams.append((bitrate, HTTPStream(self.session, video_url)))
if self.get_option("mux_subtitles") and videos['subtitles'] != []:
substreams = {}
for subtitle in videos['subtitles']:
# TTML subtitles are available but not supported by FFmpeg
if subtitle['format'] == 'ttml':
continue
substreams[subtitle['type']] = HTTPStream(
self.session, subtitle['url'])
for quality, stream in streams:
yield quality, MuxedStream(self.session,
stream,
subtitles=substreams)
else:
for stream in streams:
yield stream
if offline:
log.error('Failed to access stream, may be due to offline content')
if geolocked:
log.error(
'Failed to access stream, may be due to geo-restricted content'
)
if drm:
log.error(
'Failed to access stream, may be due to DRM-protected content')
if expired:
log.error('Failed to access stream, may be due to expired content')
def ytdl_fallback(self):
'''Basic support for m3u8 URLs with youtube-dl'''
log.debug(f'Fallback {youtube_dl.__name__} {youtube_dl.version.__version__}')
class YTDL_Logger(object):
def debug(self, msg):
log.debug(msg)
def warning(self, msg):
log.warning(msg)
def error(self, msg):
log.trace(msg)
ydl_opts = {
'call_home': False,
'forcejson': True,
'logger': YTDL_Logger(),
'no_color': True,
'noplaylist': True,
'no_warnings': True,
'verbose': False,
'quiet': True,
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
try:
info = ydl.extract_info(self.url, download=False)
except Exception:
return
if not info or not info.get('formats'):
return
self.title = info['title']
streams = []
for stream in info['formats']:
if stream['protocol'] in ['m3u8', 'm3u8_native'] and stream['ext'] == 'mp4':
log.trace('{0!r}'.format(stream))
name = stream.get('height') or stream.get('width')
if name:
name = '{0}p'.format(name)
streams.append((name, HLSStream(self.session,
stream['url'],
headers=stream['http_headers'])))
if not streams:
if ('youtube.com' in self.url
and info.get('requested_formats')
and len(info.get('requested_formats')) == 2
and MuxedStream.is_usable(self.session)):
audio_url = audio_format = video_url = video_format = video_name = None
for stream in info.get('requested_formats'):
if stream.get('format_id') == '135':
url = stream.get('manifest_url')
if not url:
return
return DASHStream.parse_manifest(self.session, url).items()
if not stream.get('height'):
audio_url = stream.get('url')
audio_format = stream.get('format_id')
if stream.get('height'):
video_url = stream.get('url')
video_format = stream.get('format_id')
video_name = '{0}p'.format(stream.get('height'))
log.debug('MuxedStream: v {video} a {audio} = {name}'.format(
audio=audio_format,
name=video_name,
video=video_format,
))
streams.append((video_name,
MuxedStream(self.session,
HTTPStream(self.session, video_url, headers=stream['http_headers']),
HTTPStream(self.session, audio_url, headers=stream['http_headers']))
))
return streams
def _resolve_playlist(self, playlist_all):
playlist_referer = self.get_option('playlist_referer') or self.url
self.session.http.headers.update({'Referer': playlist_referer})
playlist_max = self.get_option('playlist_max') or 5
count_playlist = {
'dash': 0,
'hls': 0,
'http': 0,
}
o = urlparse(self.url)
origin_tuple = (
'.cloudfront.net',
)
for url in playlist_all:
parsed_url = urlparse(url)
if parsed_url.netloc.endswith(origin_tuple):
self.session.http.headers.update({
'Origin': '{0}://{1}'.format(o.scheme, o.netloc),
})
if (parsed_url.path.endswith(('.m3u8'))
or parsed_url.query.endswith(('.m3u8'))):
if count_playlist['hls'] >= playlist_max:
log.debug('Skip - {0}'.format(url))
continue
try:
streams = HLSStream.parse_variant_playlist(self.session, url).items()
if not streams:
yield 'live', HLSStream(self.session, url)
for s in streams:
yield s
log.debug('HLS URL - {0}'.format(url))
count_playlist['hls'] += 1
except Exception as e:
log.error('Skip HLS with error {0}'.format(str(e)))
elif (parsed_url.path.endswith(('.mp3', '.mp4'))
or parsed_url.query.endswith(('.mp3', '.mp4'))):
if count_playlist['http'] >= playlist_max:
log.debug('Skip - {0}'.format(url))
continue
try:
name = 'vod'
m = self._httpstream_bitrate_re.search(url)
if m:
bitrate = m.group('bitrate')
resolution = m.group('resolution')
if bitrate:
if bitrate in self._httpstream_common_resolution_list:
name = '{0}p'.format(m.group('bitrate'))
else:
name = '{0}k'.format(m.group('bitrate'))
elif resolution:
name = resolution
yield name, HTTPStream(self.session, url)
log.debug('HTTP URL - {0}'.format(url))
count_playlist['http'] += 1
except Exception as e:
log.error('Skip HTTP with error {0}'.format(str(e)))
elif (parsed_url.path.endswith(('.mpd'))
or parsed_url.query.endswith(('.mpd'))):
if count_playlist['dash'] >= playlist_max:
log.debug('Skip - {0}'.format(url))
continue
try:
for s in DASHStream.parse_manifest(self.session,
url).items():
yield s
log.debug('DASH URL - {0}'.format(url))
count_playlist['dash'] += 1
except Exception as e:
log.error('Skip DASH with error {0}'.format(str(e)))
else:
log.error('parsed URL - {0}'.format(url))
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
log.debug('Version 2018-07-12')
log.info('This is a custom plugin. ')
match = self._url_re.match(self.url)
username = match.group('username')
user_id = match.group('user_id')
servers = self._get_servers()
chat_servers = servers['chat_servers']
message, php_message = self._websocket_data(username, chat_servers)
if user_id and not username:
data = self._php_fallback(username, user_id, php_message)
else:
log.debug('Attempting to use WebSocket data')
data = self._dict_re.search(message)
if data is None:
raise NoStreamsError(self.url)
data = parse_json(data.group('data'), schema=self._data_schema)
vs = data['vs']
ok_vs = [0, 90]
if vs not in ok_vs:
if vs == 2:
log.info('Model is currently away')
elif vs == 12:
log.info('Model is currently in a private show')
elif vs == 13:
log.info('Model is currently in a group show')
elif vs == 127:
log.info('Model is currently offline')
else:
log.error('Stream status: {0}'.format(vs))
raise NoStreamsError(self.url)
log.debug('VS: {0}'.format(vs))
nm = data['nm']
uid = data['uid']
uid_video = uid + 100000000
camserver = data['u']['camserv']
server, server_type = self._get_camserver(servers, camserver)
if server is None and not user_id:
fallback_data = self._php_fallback(username, user_id, php_message)
camserver = fallback_data['u']['camserv']
server, server_type = self._get_camserver(servers, camserver)
log.info('Username: {0}'.format(nm))
log.info('User ID: {0}'.format(uid))
if not server:
raise PluginError('Missing video server')
log.debug('Video server: {0}'.format(server))
log.debug('Video server_type: {0}'.format(server_type))
if server_type == 'h5video_servers':
DASH_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_{1}.f4v_desktop/manifest.mpd'.format(
server, uid_video)
HLS_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_{1}.f4v_mobile/playlist.m3u8'.format(
server, uid_video)
elif server_type == 'wzobs_servers':
DASH_VIDEO_URL = ''
HLS_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_a_{1}.f4v_mobile/playlist.m3u8'.format(
server, uid_video)
elif server_type == 'ngvideo_servers':
raise PluginError('ngvideo_servers are not supported.')
else:
raise PluginError('Unknow server type.')
log.debug('HLS URL: {0}'.format(HLS_VIDEO_URL))
for s in HLSStream.parse_variant_playlist(self.session,
HLS_VIDEO_URL).items():
yield s
if DASH_VIDEO_URL and self.get_option('dash'):
log.debug('DASH URL: {0}'.format(DASH_VIDEO_URL))
for s in DASHStream.parse_manifest(self.session,
DASH_VIDEO_URL).items():
yield s
def _get_streams(self):
# Retrieve geolocation data
res = self.session.http.get(self.GEO_URL)
geo = self.session.http.json(res, schema=self._geo_schema)
country_code = geo['reponse']['geo_info']['country_code']
# Retrieve URL page and search for video ID
res = self.session.http.get(self.url)
if 'france.tv' in self.url:
match = self._pluzz_video_id_re.search(res.text)
elif 'ludo.fr' in self.url or 'zouzous.fr' in self.url:
match = self._jeunesse_video_id_re.search(res.text)
elif 'sport.francetvinfo.fr' in self.url:
match = self._sport_video_id_re.search(res.text)
else:
match = self._embed_video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
res = self.session.http.get(self.API_URL.format(video_id))
videos = self.session.http.json(res, schema=self._api_schema)
now = time.time()
offline = False
geolocked = False
drm = False
expired = False
streams = []
for video in videos['videos']:
video_url = video['url']
# Check whether video format is available
if video['statut'] != 'ONLINE':
offline = offline or True
continue
# Check whether video format is geo-locked
if video['geoblocage'] is not None and country_code not in video['geoblocage']:
geolocked = geolocked or True
continue
# Check whether video is DRM-protected
if video['drm']:
drm = drm or True
continue
# Check whether video format is expired
available = False
for interval in video['plages_ouverture']:
available = (interval['debut'] or 0) <= now <= (interval['fin'] or sys.maxsize)
if available:
break
if not available:
expired = expired or True
continue
res = self.session.http.get(self.TOKEN_URL.format(video_url))
video_url = res.text
if '.mpd' in video_url:
# Get redirect video URL
res = self.session.http.get(res.text)
video_url = res.url
for bitrate, stream in DASHStream.parse_manifest(self.session,
video_url).items():
streams.append((bitrate, stream))
elif '.f4m' in video_url:
for bitrate, stream in HDSStream.parse_manifest(self.session,
video_url,
is_akamai=True,
pvswf=self.SWF_PLAYER_URL).items():
# HDS videos with data in their manifest fragment token
# doesn't seem to be supported by HDSStream. Ignore such
# stream (but HDS stream having only the hdntl parameter in
# their manifest token will be provided)
pvtoken = stream.request_params['params'].get('pvtoken', '')
match = self._hds_pv_data_re.search(pvtoken)
if match is None:
streams.append((bitrate, stream))
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(self.session, video_url).items():
streams.append(stream)
# HBB TV streams are not provided anymore by France Televisions
elif '.mp4' in video_url and '/hbbtv/' not in video_url:
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
# Fallback bitrate (seems all France Televisions MP4 videos
# seem have such bitrate)
bitrate = '1500k'
streams.append((bitrate, HTTPStream(self.session, video_url)))
if self.get_option("mux_subtitles") and videos['subtitles'] != []:
substreams = {}
for subtitle in videos['subtitles']:
# TTML subtitles are available but not supported by FFmpeg
if subtitle['format'] == 'ttml':
continue
substreams[subtitle['type']] = HTTPStream(self.session, subtitle['url'])
for quality, stream in streams:
yield quality, MuxedStream(self.session, stream, subtitles=substreams)
else:
for stream in streams:
yield stream
if offline:
self.logger.error('Failed to access stream, may be due to offline content')
if geolocked:
self.logger.error('Failed to access stream, may be due to geo-restricted content')
if drm:
self.logger.error('Failed to access stream, may be due to DRM-protected content')
if expired:
self.logger.error('Failed to access stream, may be due to expired content')
def test_parse_manifest_string(self):
with text("dash/test_9.mpd") as mpd_txt:
test_manifest = mpd_txt.read()
streams = DASHStream.parse_manifest(self.session, test_manifest)
self.assertSequenceEqual(list(streams.keys()), ['2500k'])
def _get_streams(self):
if self.response is None:
infos = self.session.http.get(self.url, schema=self._ctcomp_schema)
else:
infos = self.session.http.json(self.response,
schema=self._ctcomp_schema)
if not infos:
# playlist infos not found
raise PluginError('Cannot find playlist infos!')
vod_prio = len(infos) == 2
for info in infos:
try:
pl = info['ctcomp-data']['source']['playlist'][0]
except KeyError:
raise PluginError('Cannot find playlist info!')
pl = self._playlist_info_schema.validate(pl)
if vod_prio and pl['type'] != 'VOD':
continue
log.trace('{0!r}'.format(info))
if pl['type'] == 'LIVE':
data = {
"contentType":
"live",
"items": [{
"id": pl["id"],
"assetId": pl["assetId"],
"key": pl["key"],
"playerType": "dash",
"date": pl["date"],
"requestSource": pl["requestSource"],
"drm": pl["drm"],
"quality": pl["quality"],
}]
}
elif pl['type'] == 'VOD':
data = {
"contentType":
"vod",
"items": [{
"id": pl["id"],
"key": pl["key"],
"playerType": "dash",
"date": pl["date"],
"requestSource": pl["requestSource"],
"drm": pl["drm"],
"canBePlay": pl["canBePlay"],
"quality": pl["quality"],
"region": pl["region"]
}]
}
headers = {
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
}
data = json.dumps(data)
response = self.session.http.post(self._player_api,
data="data={}".format(quote(data)),
headers=headers)
json_data = self.session.http.json(response,
schema=self._playlist_schema)
log.trace('{0!r}'.format(json_data))
playlist = json_data['RESULT']['playlist'][0]['streamUrls']['main']
for s in DASHStream.parse_manifest(self.session, playlist).items():
yield s
def _watch(self):
log.debug('_watch ...')
match = self._url_re.match(self.url)
if not match:
log.debug('_watch ... no match')
return
channel = match.group('channel')
vod_id = match.group('vod_id')
recording_id = match.group('recording_id')
params = {'https_watch_urls': True}
if channel:
watch_url = self.API_WATCH.format(self.base_url)
params_cid = self._get_params_cid(channel)
if not params_cid:
return
params.update(params_cid)
elif vod_id:
log.debug('Found vod_id: {0}'.format(vod_id))
watch_url = self.API_WATCH_VOD.format(self.base_url, vod_id)
elif recording_id:
log.debug('Found recording_id: {0}'.format(recording_id))
watch_url = self.API_WATCH_REC.format(self.base_url, recording_id)
else:
log.debug('Missing watch_url')
return
zattoo_stream_types = self.get_option('stream-types') or ['hls']
for stream_type in zattoo_stream_types:
params_stream_type = {'stream_type': stream_type}
params.update(params_stream_type)
try:
res = self.session.http.post(watch_url, headers=self.headers, data=params)
except Exception as e:
if '404 Client Error' in str(e):
log.error('Unfortunately streaming is not permitted in '
'this country or this channel does not exist.')
elif '402 Client Error: Payment Required' in str(e):
log.error('Paid subscription required for this channel.')
log.info('If paid subscription exist, use --zattoo-purge'
'-credentials to start a new session.')
elif '403 Client Error' in str(e):
log.debug('Force session reset for watch_url')
self.reset_session()
else:
log.error(str(e))
return
data = self.session.http.json(res)
log.debug('Found data for {0}'.format(stream_type))
if data['success'] and stream_type in ['hls', 'hls5']:
for url in data['stream']['watch_urls']:
for s in HLSStream.parse_variant_playlist(
self.session, url['url']).items():
yield s
elif data['success'] and stream_type == 'dash':
for url in data['stream']['watch_urls']:
for s in DASHStream.parse_manifest(
self.session, url['url']).items():
yield s