def _get_streams(self):
m = self.url_re.match(self.url)
episode_id = m.group("episode_id")
channel_name = m.group("channel_name")
if episode_id:
self.logger.debug("Loading streams for episode: {0}", episode_id)
vpid = self.find_vpid(self.url)
if vpid:
self.logger.debug("Found VPID: {0}", vpid)
s = self.mediaselector(vpid)
for url in s:
for s in HDSStream.parse_manifest(self.session,
url).items():
yield s
else:
self.logger.error("Could not find VPID for episode {0}",
episode_id)
elif channel_name:
self.logger.debug("Loading stream for live channel: {0}",
channel_name)
tvip = self.find_tvip(self.url)
if tvip:
self.logger.debug("Found TVIP: {0}", tvip)
s = self.mediaselector(tvip)
for url in s:
for s in HDSStream.parse_manifest(self.session,
url).items():
yield s
def test_hds_stream(self):
stream = HDSStream(self.session, "http://test.se/", "http://test.se/stream.f4m",
"http://test.se/stream/1.bootstrap", headers={"User-Agent": "Test"})
self.assertEqual(
{"type": "hds",
"baseurl": "http://test.se/",
"bootstrap": "http://test.se/stream/1.bootstrap",
"url": "http://test.se/stream.f4m",
"metadata": None,
"headers": {"User-Agent": "Test"},
"params": {}},
stream.__json__()
)
def _get_streams(self):
match = _url_re.match(self.url)
video_id = match.group("video_id")
res = http.get(ASSET_URL.format(video_id))
assets = http.xml(res, schema=_asset_schema)
streams = {}
for asset in assets:
base = asset["base"]
url = asset["url"]
if urlparse(url).path.endswith(".f4m"):
streams.update(
HDSStream.parse_manifest(self.session, url, pvswf=SWF_URL)
)
elif base.startswith("rtmp"):
name = "{0}k".format(asset["bitrate"])
params = {
"rtmp": asset["base"],
"playpath": url,
"live": True
}
streams[name] = RTMPStream(self.session, params)
return streams
def _get_streams(self):
match = self._url_re.match(self.url)
channel = match.group('channel')
res = self.session.http.get(self.FORMATS_URL.format(channel))
streams = self.session.http.json(res, schema=self._formats_schema)['streams']
if streams == []:
self.logger.error('Channel may be geo-restricted, not directly provided by PlayTV or not freely available')
return
for language in streams:
for protocol, bitrates in list(streams[language].items()):
# - Ignore non-supported protocols (RTSP, DASH)
# - Ignore deprecated Flash (RTMPE/HDS) streams (PlayTV doesn't provide anymore a Flash player)
if protocol in ['rtsp', 'flash', 'dash', 'hds']:
continue
for bitrate in bitrates['bitrates']:
if bitrate['value'] == 0:
continue
api_url = self.API_URL.format(channel, protocol, language, bitrate['value'])
res = self.session.http.get(api_url)
video_url = self.session.http.json(res, schema=self._api_schema)['url']
bs = '{0}k'.format(bitrate['value'])
if protocol == 'hls':
for _, stream in HLSStream.parse_variant_playlist(self.session, video_url).items():
yield bs, stream
elif protocol == 'hds':
for _, stream in HDSStream.parse_manifest(self.session, video_url).items():
yield bs, stream
def _get_streams(self):
data_url = self.session.http.get(self.url,
schema=self._player_url_schema)
if data_url:
res = self.session.http.get(urljoin(self.url, data_url))
stream_info = self.session.http.xml(res,
schema=self._livestream_schema)
for stream in stream_info:
url = stream["url"]
try:
if ".m3u8" in url:
for s in HLSStream.parse_variant_playlist(
self.session, url, name_key="bitrate").items():
yield s
elif ".f4m" in url:
for s in HDSStream.parse_manifest(
self.session,
url,
pvswf=self.swf_url,
is_akamai=True).items():
yield s
elif ".mp4" in url:
yield "{0}k".format(stream["bitrate"]), HTTPStream(
self.session, url)
except IOError as err:
self.logger.warning("Error parsing stream: {0}", err)
def _create_streams(self, type_, video_id):
url = self._generate_security_url(type_, video_id)
res = http.get(url)
return HDSStream.parse_manifest(self.session,
res.text,
cookies=res.cookies)
def mediaselector(self, vpid):
urls = defaultdict(set)
for platform in self.platforms:
url = self.api_url.format(vpid=vpid,
vpid_hash=self._hash_vpid(vpid),
platform=platform)
self.logger.debug("Info API request: {0}", url)
medias = http.get(url, schema=self.mediaselector_schema)
for media in medias:
for connection in media["connection"]:
urls[connection.get("transferFormat")].add(
connection["href"])
for stream_type, urls in urls.items():
self.logger.debug("{0} {1} streams", len(urls), stream_type)
for url in list(urls):
self.logger.debug(" {0}", url)
if stream_type == "hds":
for s in HDSStream.parse_manifest(self.session,
url).items():
yield s
if stream_type == "hls":
for s in HLSStream.parse_variant_playlist(
self.session, url).items():
yield s
if connection.get("transferFormat") == "dash":
for s in DASHStream.parse_manifest(
self.session, connection["href"]).items():
yield s
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme("http://", urlnoproto)
return HDSStream.parse_manifest(self.session, urlnoproto, **params)
def _get_vod_stream(self):
vod_url = self.url
if vod_url.endswith('/'):
vod_url = vod_url[:-1]
json_url = '{0}.securevideo.json'.format(vod_url)
res = http.get(json_url)
match = _json_re.search(res.text)
if not match:
return
data = parse_json(match.group(1))
res = http.get(API_VOD.format(data['clientid'], data['mzid']))
data = http.json(res, schema=_stream_schema)
for d in data['targetUrls']:
if d['type'] == 'HDS':
hds_url = d['url']
for s in HDSStream.parse_manifest(self.session, hds_url).items():
yield s
if d['type'] == 'HLS':
hls_url = d['url']
for s in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield s
def _get_live_streams(self, params, swf_url):
for key, quality in QUALITY_MAP.items():
key_url = "{0}URL".format(key)
url = params.get(key_url)
if not url:
continue
try:
res = http.get(url, exception=IOError)
except IOError:
continue
if quality == "hds":
streams = HDSStream.parse_manifest(self.session, res.url)
for name, stream in streams.items():
if key == "source":
name += "+"
yield name, stream
elif res.text.startswith("rtmp"):
match = _rtmp_re.match(res.text)
if not match:
continue
stream = RTMPStream(self.session, {
"rtmp": match.group("host"),
"app": match.group("app"),
"playpath": match.group("playpath"),
"swfVfy": swf_url,
"live": True
})
yield quality, stream
def _get_streams(self):
res = http.get(self.url)
match = _player_js.search(res.text)
if match:
player_js = match.group(0)
self.logger.info("Found player js {0}", player_js)
else:
self.logger.info(
"Didn't find player js. Probably this page doesn't contain a video"
)
return
res = http.get(player_js)
jsonp_start = res.text.find('(') + 1
jsonp_end = res.text.rfind(')')
if jsonp_start <= 0 or jsonp_end <= 0:
self.logger.info(
"Couldn't extract json metadata from player.js: {0}",
player_js)
return
json_s = res.text[jsonp_start:jsonp_end]
stream_metadata = json.loads(json_s)
return HDSStream.parse_manifest(
self.session,
stream_metadata['mediaResource']['dflt']['videoURL']).items()
def _get_vod_streams(self, params):
manifest_url = params.get("autoURL")
if not manifest_url:
return
res = http.get(manifest_url)
if res.headers.get("Content-Type") == "application/f4m+xml":
streams = HDSStream.parse_manifest(self.session, res.url)
# TODO: Replace with "yield from" when dropping Python 2.
for __ in streams.items():
yield __
elif res.headers.get("Content-Type") == "application/vnd.apple.mpegurl":
streams = HLSStream.parse_variant_playlist(self.session, res.url)
# TODO: Replace with "yield from" when dropping Python 2.
for __ in streams.items():
yield __
else:
manifest = http.json(res, schema=_vod_manifest_schema)
for params in manifest["alternates"]:
name = "{0}p".format(params["height"])
stream = self._create_flv_playlist(params["template"])
yield name, stream
failovers = params.get("failover", [])
for failover in failovers:
stream = self._create_flv_playlist(failover)
yield name, stream
def _get_streams(self):
url, params = parse_url_params(self.url)
urlnoproto = self._url_re.match(url).group(2)
urlnoproto = update_scheme("http://", urlnoproto)
return HDSStream.parse_manifest(self.session, urlnoproto, **params)
def _get_vod_streams(self, params):
manifest_url = params.get("autoURL")
if not manifest_url:
return
res = http.get(manifest_url)
if res.headers.get("Content-Type") == "application/f4m+xml":
streams = HDSStream.parse_manifest(self.session, res.url)
# TODO: Replace with "yield from" when dropping Python 2.
for __ in streams.items():
yield __
elif res.headers.get("Content-Type") == "application/vnd.apple.mpegurl":
streams = HLSStream.parse_variant_playlist(self.session, res.url)
# TODO: Replace with "yield from" when dropping Python 2.
for __ in streams.items():
yield __
else:
manifest = http.json(res, schema=_vod_manifest_schema)
for params in manifest["alternates"]:
name = "{0}p".format(params["height"])
stream = self._create_flv_playlist(params["template"])
yield name, stream
failovers = params.get("failover", [])
for failover in failovers:
stream = self._create_flv_playlist(failover)
yield name, stream
def _get_streams(self):
match = self._url_re.match(self.url)
channel = match.group('channel')
res = http.get(self.FORMATS_URL.format(channel))
streams = http.json(res, schema=self._formats_schema)['streams']
if streams == []:
self.logger.error('Channel may be geo-restricted, not directly provided by PlayTV or not freely available')
return
for language in streams:
for protocol, bitrates in list(streams[language].items()):
# - Ignore non-supported protocols (RTSP, DASH)
# - Ignore deprecated Flash (RTMPE) streams (PlayTV doesn't provide anymore a Flash player)
if protocol in ['rtsp', 'flash', 'dash']:
continue
for bitrate in bitrates['bitrates']:
if bitrate['value'] == 0:
continue
api_url = self.API_URL.format(channel, protocol, language, bitrate['value'])
res = http.get(api_url)
video_url = http.json(res, schema=self._api_schema)['url']
bs = '{0}k'.format(bitrate['value'])
if protocol == 'hls':
for _, stream in HLSStream.parse_variant_playlist(self.session, video_url).items():
yield bs, stream
elif protocol == 'hds':
for _, stream in HDSStream.parse_manifest(self.session, video_url).items():
yield bs, stream
def mediaselector(self, vpid):
urls = defaultdict(set)
for platform in self.platforms:
url = self.api_url.format(vpid=vpid, vpid_hash=self._hash_vpid(vpid),
platform=platform)
log.debug("Info API request: {0}", url)
medias = self.session.http.get(url, schema=self.mediaselector_schema)
for media in medias:
for connection in media["connection"]:
urls[connection.get("transferFormat")].add(connection["href"])
for stream_type, urls in urls.items():
log.debug("{0} {1} streams", len(urls), stream_type)
for url in list(urls):
try:
if stream_type == "hds":
for s in HDSStream.parse_manifest(self.session,
url).items():
yield s
if stream_type == "hls":
for s in HLSStream.parse_variant_playlist(self.session,
url).items():
yield s
if stream_type == "dash":
for s in DASHStream.parse_manifest(self.session,
url).items():
yield s
log.debug(" OK: {0}", url)
except:
log.debug(" FAIL: {0}", url)
def _get_vod_stream(self):
vod_url = self.url
if vod_url.endswith('/'):
vod_url = vod_url[:-1]
json_url = '{0}.securevideo.json'.format(vod_url)
res = http.get(json_url)
match = _json_re.search(res.text)
if not match:
return
data = parse_json(match.group(1))
res = http.get(API_VOD.format(data['clientid'], data['mzid']))
data = http.json(res, schema=_stream_schema)
for d in data['targetUrls']:
if d['type'] == 'HDS':
hds_url = d['url']
for s in HDSStream.parse_manifest(self.session,
hds_url).items():
yield s
if d['type'] == 'HLS':
hls_url = d['url']
for s in HLSStream.parse_variant_playlist(
self.session, hls_url).items():
yield s
def _get_streams(self):
# Get video ID and channel from URL
match = self._url_re.match(self.url)
video_id = match.group('video_id')
if video_id is None:
# Retrieve URL page and search for video ID
res = self.session.http.get(self.url)
match = self._video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
res = self.session.http.get(self.API_URL.format(video_id))
videos = self.session.http.json(res, schema=self._api_schema)
parsed = []
headers = {'User-Agent': self._user_agent}
# Some videos may be also available on Dailymotion (especially on CNews)
if videos['ID_DM'] != '':
yield from self.session.streams('https://www.dailymotion.com/video/' + videos['ID_DM']).items()
for quality, video_url in list(videos['MEDIA']['VIDEOS'].items()):
# Ignore empty URLs
if video_url == '':
continue
# Ignore duplicate video URLs
if video_url in parsed:
continue
parsed.append(video_url)
try:
# HDS streams don't seem to work for live videos
if '.f4m' in video_url and 'LIVE' not in videos['TYPE']:
yield from HDSStream.parse_manifest(
self.session,
video_url,
params={'hdcore': self.HDCORE_VERSION},
headers=headers
).items()
elif '.m3u8' in video_url:
yield from HLSStream.parse_variant_playlist(
self.session,
video_url,
headers=headers
).items()
elif '.mp4' in video_url:
# Get bitrate from video filename
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
bitrate = quality
yield bitrate, HTTPStream(self.session,
video_url,
params={'secret': self.SECRET},
headers=headers)
except OSError as err:
if '403 Client Error' in str(err):
log.error('Failed to access stream, may be due to geo-restriction')
def mediaselector(self, vpid):
urls = defaultdict(set)
for platform in self.platforms:
url = self.api_url.format(vpid=vpid,
vpid_hash=self._hash_vpid(vpid),
platform=platform)
log.debug(f"Info API request: {url}")
medias = self.session.http.get(url,
schema=self.mediaselector_schema)
for media in medias:
for connection in media["connection"]:
urls[connection.get("transferFormat")].add(
connection["href"])
for stream_type, urls in urls.items():
log.debug(f"{len(urls)} {stream_type} streams")
for url in list(urls):
try:
if stream_type == "hds":
yield from HDSStream.parse_manifest(self.session,
url).items()
if stream_type == "hls":
yield from HLSStream.parse_variant_playlist(
self.session, url).items()
if stream_type == "dash":
yield from DASHStream.parse_manifest(
self.session, url).items()
log.debug(f" OK: {url}")
except Exception:
log.debug(f" FAIL: {url}")
def _get_streams(self):
res = self.session.http.get(self.url)
match = _player_js.search(res.text)
if match:
player_js = match.group(0)
self.logger.info("Found player js {0}", player_js)
else:
self.logger.info("Didn't find player js. Probably this page doesn't contain a video")
return
res = self.session.http.get(player_js)
jsonp_start = res.text.find('(') + 1
jsonp_end = res.text.rfind(')')
if jsonp_start <= 0 or jsonp_end <= 0:
self.logger.info("Couldn't extract json metadata from player.js: {0}", player_js)
return
json_s = res.text[jsonp_start:jsonp_end]
stream_metadata = json.loads(json_s)
hds_url = stream_metadata['mediaResource']['dflt']['videoURL']
hds_url = update_scheme(self.url, hds_url)
return HDSStream.parse_manifest(self.session, hds_url).items()
def _get_streams(self):
# Discover root
match = _url_re.search(self.url)
root = match.group(1)
# Download main URL
res = http.get(self.url)
# Find playlist
match = _playlist_re.search(res.text)
playlist_url = root + match.group(1) + "d"
# Download playlist
res = http.get(playlist_url)
# Find manifest
match = _manifest_re.search(res.text)
manifest_url = match.group(1)
# Find SWF
match = _swf_re.search(res.text)
swf_url = match.group(1)
streams = {}
streams.update(
HDSStream.parse_manifest(self.session, manifest_url, pvswf=swf_url)
)
return streams
def _get_streams(self):
match = self._url_re.match(self.url)
channel = match.group("channel")
self.session.http.headers.update({"User-Agent": useragents.CHROME})
if channel:
streams = self._get_live_streams(channel) or []
else:
streams = self._get_vod_streams() or []
for video_url in streams:
log.debug("Found stream: {0}".format(video_url))
parsed = urlparse(video_url)
if parsed.path.endswith(".f4m"):
for s in HDSStream.parse_manifest(self.session,
video_url).items():
yield s
elif parsed.path.endswith(".m3u8"):
for s in HLSStream.parse_variant_playlist(
self.session, video_url).items():
yield s
elif parsed.path.endswith(".mp4"):
match = self._re_mp4_bitrate.match(video_url)
bitrate = "vod" if match is None else "{0}k".format(
match.group('bitrate'))
yield bitrate, HTTPStream(self.session, video_url)
def _get_live_streams(self, params, swf_url):
for key, quality in QUALITY_MAP.items():
key_url = "{0}URL".format(key)
url = params.get(key_url)
if not url:
continue
try:
res = http.get(url, exception=IOError)
except IOError:
continue
if quality == "hds":
streams = HDSStream.parse_manifest(self.session, res.url)
for name, stream in streams.items():
if key == "source":
name += "+"
yield name, stream
elif res.text.startswith("rtmp"):
match = _rtmp_re.match(res.text)
if not match:
continue
stream = RTMPStream(self.session, {
"rtmp": match.group("host"),
"app": match.group("app"),
"playpath": match.group("playpath"),
"swfVfy": swf_url,
"live": True
})
yield quality, stream
def _get_streams(self):
self.session.http.headers.update({"User-Agent": useragents.CHROME})
if '/news/videos/' in self.url:
# VOD
streams = self._get_vod_streams()
else:
# Live
streams = self._get_live_streams()
for video_url in streams:
log.debug("Found stream: {0}".format(video_url))
if '.f4m' in video_url:
for stream in HDSStream.parse_manifest(self.session,
video_url).items():
yield stream
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(
self.session, video_url).items():
yield stream
if '.mp4' in video_url:
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate') + 'k'
else:
bitrate = 'vod'
yield bitrate, HTTPStream(self.session, video_url)
def _get_streams(self):
data = self.match.groupdict()
url = update_scheme("http://", data.get("url"))
params = parse_params(data.get("params"))
log.debug(f"URL={url}; params={params}")
return HDSStream.parse_manifest(self.session, url, **params)
def _get_star_streams(self, desktop_url, mobile_url, token=""):
if token:
self.logger.debug("Opening stream with token: {}", token)
if mobile_url:
for _, s in HLSStream.parse_variant_playlist(self.session,
mobile_url + token,
headers={
"Referer":
self.url
}).items():
yield "live", s
if desktop_url:
# get the HDS stream URL
res = http.get(desktop_url + token)
stream_data = http.json(res, schema=self.hds_schema)
for _, s in HDSStream.parse_manifest(
self.session,
stream_data["url"],
pvswf=self.SWF_URL,
is_akamai=stream_data["use_akamai"],
headers={
"Referer": self.url
}).items():
yield "live", s
def _get_streams(self):
# Get video ID and channel from URL
match = self._url_re.match(self.url)
video_id = match.group('video_id')
if video_id is None:
# Retrieve URL page and search for video ID
res = http.get(self.url)
match = self._video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
res = http.get(self.API_URL.format(video_id))
videos = http.json(res, schema=self._api_schema)
parsed = []
headers = {'User-Agent': self._user_agent}
# Some videos may be also available on Dailymotion (especially on CNews)
if videos['ID_DM'] != '':
for stream in self.session.streams('https://www.dailymotion.com/video/' + videos['ID_DM']).items():
yield stream
for quality, video_url in list(videos['MEDIA']['VIDEOS'].items()):
# Ignore empty URLs
if video_url == '':
continue
# Ignore duplicate video URLs
if video_url in parsed:
continue
parsed.append(video_url)
try:
# HDS streams don't seem to work for live videos
if '.f4m' in video_url and 'LIVE' not in videos['TYPE']:
for stream in HDSStream.parse_manifest(self.session,
video_url,
params={'hdcore': self.HDCORE_VERSION},
headers=headers).items():
yield stream
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(self.session,
video_url,
headers=headers).items():
yield stream
elif '.mp4' in video_url:
# Get bitrate from video filename
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
bitrate = quality
yield bitrate, HTTPStream(self.session,
video_url,
params={'secret': self.SECRET},
headers=headers)
except IOError as err:
if '403 Client Error' in str(err):
self.logger.error('Failed to access stream, may be due to geo-restriction')
def _get_streams(self):
# Get video ID and channel from URL
match = self._url_re.match(self.url)
channel = match.group('channel')
if channel is None:
# Replay website
channel = match.group('replay_channel')
video_id = match.group('replay_video_id')
else:
video_id = match.group('video_id')
if video_id is None:
# Retrieve URL page and search for video ID
res = http.get(self.url)
match = self._video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
res = http.get(self.API_URL.format(self.CHANNEL_MAP[channel], video_id))
videos = http.json(res, schema=self._api_schema)
parsed = []
headers = {'User-Agent': self._user_agent}
for quality, video_url in list(videos['MEDIA']['VIDEOS'].items()):
# Ignore empty URLs
if video_url == '':
continue
# Ignore duplicate video URLs
if video_url in parsed:
continue
parsed.append(video_url)
try:
# HDS streams don't seem to work for live videos
if '.f4m' in video_url and videos['TYPE'] != 'CHAINE LIVE':
for stream in HDSStream.parse_manifest(self.session,
video_url,
params={'hdcore': self.HDCORE_VERSION},
headers=headers).items():
yield stream
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(self.session,
video_url,
headers=headers).items():
yield stream
elif '.mp4' in video_url:
# Get bitrate from video filename
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
bitrate = quality
yield bitrate, HTTPStream(self.session,
video_url,
params={'secret': self.SECRET},
headers=headers)
except PluginError:
self.logger.error('Failed to access stream, may be due to geo-restriction')
raise
def _get_streams(self):
url_match = _url_re.match(self.url)
if url_match:
if url_match.group(1) in _channel:
return HDSStream.parse_manifest(
self.session,
STREAM_INFO_URL.format(
channel=_channel[url_match.group(1)]))
def _get_smil_streams(self, info):
res = http.get(info["_stream"])
smil = http.xml(res, "SMIL config", schema=_smil_schema)
for video in smil["videos"]:
url = "{0}/{1}{2}".format(smil["base"], video, HDCORE_PARAMETER)
streams = HDSStream.parse_manifest(self.session, url, pvswf=SWF_URL, is_akamai=smil["cdn"] == "akamai")
for stream in streams.items():
yield stream
def _get_streams(self):
res = http.get(self.url)
match = _manifest_re.search(res.text)
manifest = match.group(1)
streams = {}
streams.update(
HDSStream.parse_manifest(self.session, manifest, pvswf=SWF_URL)
)
return streams
def _get_hds_streams(self, channel):
channel = self.hds_channel_remap.get(channel, "{0}live".format(channel))
manifest_url = http.get(self.api_url.format(channel),
params={"getURL": 1},
headers={"User-Agent": useragents.FIREFOX}).text
for s in HDSStream.parse_manifest(self.session,
manifest_url,
pvswf=self.swf_url,
headers={"User-Agent": useragents.FIREFOX}).items():
yield s
def _get_hds_streams(self, channel):
channel = self.hds_channel_remap.get(channel, "{0}live".format(channel))
manifest_url = http.get(self.api_url.format(channel),
params={"getURL": 1},
headers={"User-Agent": useragents.FIREFOX}).text
for s in HDSStream.parse_manifest(self.session,
manifest_url,
pvswf=self.swf_url,
headers={"User-Agent": useragents.FIREFOX}).items():
yield s
def _get_smil_streams(self, info):
res = http.get(info["_stream"])
smil = http.xml(res, "SMIL config", schema=_smil_schema)
for video in smil["videos"]:
url = "{0}/{1}{2}".format(smil["base"], video, HDCORE_PARAMETER)
streams = HDSStream.parse_manifest(self.session, url, pvswf=SWF_URL)
# TODO: Replace with "yield from" when dropping Python 2.
for stream in streams.items():
yield stream
def _get_streams(self):
res = self.session.http.get(self.url)
url_m = self.m3u8_re.search(res.text)
hls_url = url_m and url_m.group("url")
if hls_url:
yield from HLSStream.parse_variant_playlist(self.session,
hls_url).items()
f4m_m = self.f4mm_re.search(res.text)
f4m_url = f4m_m and f4m_m.group("url")
if f4m_url:
yield from HDSStream.parse_manifest(self.session, f4m_url).items()
def _get_streams(self):
res = http.get(self.url)
match = _meta_xmlurl_id_re.search(res.text)
if not match:
return
xml_info_url = STREAMS_INFO_URL.format(match.group(1))
video_info_res = http.get(xml_info_url)
parsed_info = http.xml(video_info_res)
live_el = parsed_info.find("live")
live = live_el is not None and live_el.text == "1"
streams = {}
hdsurl_el = parsed_info.find("hdsurl")
if hdsurl_el is not None and hdsurl_el.text is not None:
hdsurl = hdsurl_el.text
streams.update(HDSStream.parse_manifest(self.session, hdsurl))
if live:
vurls_el = parsed_info.find("vurls")
if vurls_el is not None:
for i, vurl_el in enumerate(vurls_el):
bitrate = vurl_el.get("bitrate")
name = bitrate + "k" if bitrate is not None else "rtmp{0}".format(
i)
params = {
"rtmp": vurl_el.text,
}
streams[name] = RTMPStream(self.session, params)
parsed_urls = set()
mobileurls_el = parsed_info.find("mobileurls")
if mobileurls_el is not None:
for mobileurl_el in mobileurls_el:
text = mobileurl_el.text
if not text:
continue
if text in parsed_urls:
continue
parsed_urls.add(text)
url = urlparse(text)
if url[0] == "http" and url[2].endswith("m3u8"):
streams.update(
HLSStream.parse_variant_playlist(self.session, text))
return streams
def mediaselector(self, vpid):
for platform in self.platforms:
url = self.api_url.format(vpid=vpid, vpid_hash=self._hash_vpid(vpid), platform=platform)
self.logger.debug("Info API request: {0}", url)
stream_urls = http.get(url, schema=self.mediaselector_schema)
for media in stream_urls:
for connection in media["connection"]:
if connection.get("transferFormat") == "hds":
for s in HDSStream.parse_manifest(self.session, connection["href"]).items():
yield s
if connection.get("transferFormat") == "hls":
for s in HLSStream.parse_variant_playlist(self.session, connection["href"]).items():
yield s
def _get_streams(self):
res = http.get(API_URL)
data = http.json(res, schema=_schema)
streams = {}
for livestreams in data["live-streams"]:
for stream in livestreams["streams"]:
url = stream["streamUrl"]
for name, stream in HDSStream.parse_manifest(self.session, url).items():
if name.endswith("k"):
streams[name] = stream
return streams
def mediaselector(self, vpid):
for platform in self.platforms:
url = self.api_url.format(vpid=vpid,
vpid_hash=self._hash_vpid(vpid),
platform=platform)
stream_urls = http.get(url, schema=self.mediaselector_schema)
for surl in stream_urls.get("hls"):
for s in HLSStream.parse_variant_playlist(self.session,
surl).items():
yield s
for surl in stream_urls.get("hds"):
for s in HDSStream.parse_manifest(self.session, surl).items():
yield s
def mediaselector(self, vpid):
for platform in self.platforms:
url = self.api_url.format(vpid=vpid, vpid_hash=self._hash_vpid(vpid), platform=platform)
self.logger.debug("Info API request: {0}", url)
stream_urls = http.get(url, schema=self.mediaselector_schema)
for media in stream_urls:
for connection in media["connection"]:
if connection.get("transferFormat") == "hds":
for s in HDSStream.parse_manifest(self.session, connection["href"]).items():
yield s
if connection.get("transferFormat") == "hls":
for s in HLSStream.parse_variant_playlist(self.session, connection["href"]).items():
yield s
def _get_streams(self):
match = self._url_re.match(self.url)
video_id = match.group('video_id')
if video_id is not None:
# VOD
res = self.session.http.get(self.VOD_API_URL.format(video_id))
stream_data = self.session.http.json(res,
schema=self._vod_api_schema)
# Check whether video format is expired
current_date = datetime.strptime(stream_data['current_date'],
'%Y-%m-%dT%H:%M:%S.%f')
valid_start = datetime.strptime(
stream_data['shows']['valid_start'], '%Y-%m-%dT%H:%M:%S')
valid_end = datetime.strptime(stream_data['shows']['valid_end'],
'%Y-%m-%dT%H:%M:%S')
if current_date < valid_start or current_date > valid_end:
self.logger.error(
'Failed to access stream, may be due to expired content')
return
streams = stream_data['shows']['media:group']
else:
# Live
channel_id = match.group('channel_id')
# Get live streams for desktop
res = self.session.http.get(self.LIVE_API_URL,
params={'channelid': channel_id})
streams = self.session.http.xml(res, schema=self._live_api_schema)
# Get HLS streams for Iphone
res = self.session.http.get(self.LIVE_API_URL,
params={
'channelid': channel_id,
'platform': 'iphone'
})
stream = self.session.http.json(
res, schema=self._live_api_iphone_schema)
if stream != 'none':
streams.append(stream)
for stream in streams:
if '.f4m' in stream:
for s in HDSStream.parse_manifest(self.session,
stream).items():
yield s
if '.m3u8' in stream:
for s in HLSStream.parse_variant_playlist(
self.session, stream).items():
yield s
def _get_streams(self):
res = http.get(API_URL)
data = http.json(res, schema=_schema)
streams = {}
for livestreams in data["live-streams"]:
for stream in livestreams["streams"]:
url = stream["streamUrl"]
for name, stream in HDSStream.parse_manifest(
self.session, url).items():
if name.endswith("k"):
streams[name] = stream
return streams
def _get_streams(self):
res = http.get(self.url)
match = _meta_xmlurl_id_re.search(res.text)
if not match:
return
xml_info_url = STREAMS_INFO_URL.format(match.group(1))
video_info_res = http.get(xml_info_url)
parsed_info = http.xml(video_info_res)
live_el = parsed_info.find("live")
live = live_el is not None and live_el.text == "1"
streams = {}
hdsurl_el = parsed_info.find("hdsurl")
if hdsurl_el is not None and hdsurl_el.text is not None:
hdsurl = hdsurl_el.text
streams.update(HDSStream.parse_manifest(self.session, hdsurl))
if live:
vurls_el = parsed_info.find("vurls")
if vurls_el is not None:
for i, vurl_el in enumerate(vurls_el):
bitrate = vurl_el.get("bitrate")
name = bitrate + "k" if bitrate is not None else "rtmp{0}".format(i)
params = {
"rtmp": vurl_el.text,
}
streams[name] = RTMPStream(self.session, params)
parsed_urls = set()
mobileurls_el = parsed_info.find("mobileurls")
if mobileurls_el is not None:
for mobileurl_el in mobileurls_el:
text = mobileurl_el.text
if not text:
continue
if text in parsed_urls:
continue
parsed_urls.add(text)
url = urlparse(text)
if url[0] == "http" and url[2].endswith("m3u8"):
streams.update(HLSStream.parse_variant_playlist(self.session, text))
return streams
def _get_smil_streams(self, info):
res = http.get(info["_stream"])
smil = http.xml(res, "SMIL config", schema=_smil_schema)
for video in smil["videos"]:
url = "{0}/{1}{2}".format(smil["base"], video, HDCORE_PARAMETER)
streams = HDSStream.parse_manifest(
self.session,
url,
pvswf=SWF_URL,
is_akamai=smil["cdn"] == "akamai")
for stream in streams.items():
yield stream
def _get_streams(self):
data_url = self.session.http.get(self.url, schema=self._player_url_schema)
if data_url:
res = self.session.http.get(urljoin(self.url, data_url))
stream_info = self.session.http.xml(res, schema=self._livestream_schema)
for stream in stream_info:
url = stream["url"]
try:
if ".m3u8" in url:
for s in HLSStream.parse_variant_playlist(self.session, url, name_key="bitrate").items():
yield s
elif ".f4m" in url:
for s in HDSStream.parse_manifest(self.session, url, pvswf=self.swf_url, is_akamai=True).items():
yield s
elif ".mp4" in url:
yield "{0}k".format(stream["bitrate"]), HTTPStream(self.session, url)
except IOError as err:
self.logger.warning("Error parsing stream: {0}", err)
def _get_streams(self):
args = dict(parse_qsl(urlparse(self.url).query))
if "k" in args:
self.logger.debug("Loading channel: {k}", **args)
res = http.get(self.url)
stream_data_m = self.stream_data_re.search(res.text)
if stream_data_m:
script_vars = b64decode(stream_data_m.group(1)).decode("utf8")
url_m = self.m3u8_re.search(script_vars)
hls_url = url_m and url_m.group("url")
if hls_url:
for s in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield s
f4m_m = self.f4mm_re.search(script_vars)
f4m_url = f4m_m and f4m_m.group("url")
if f4m_url:
for n, s in HDSStream.parse_manifest(self.session, f4m_url).items():
yield n, s
def _get_streams(self):
if '/news/videos/' in self.url:
# VOD
streams = self._get_vod_streams()
else:
# Live
streams = self._get_live_streams()
for video_url in streams:
if '.f4m' in video_url:
for stream in HDSStream.parse_manifest(self.session, video_url).items():
yield stream
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(self.session, video_url).items():
yield stream
if '.mp4' in video_url:
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate') + 'k'
else:
bitrate = 'vod'
yield bitrate, HTTPStream(self.session, video_url)
def _create_stream(self, stream, language):
stream_name = "{0}p".format(stream["height"])
stream_type = stream["mediaType"]
stream_url = stream["url"]
stream_language = stream["versionShortLibelle"]
if language == "de":
language = ["DE", "VOST-DE", "VA", "VOA", "Dt. Live", "OV", "OmU"]
elif language == "en":
language = ["ANG", "VOST-ANG"]
elif language == "es":
language = ["ESP", "VOST-ESP"]
elif language == "fr":
language = ["FR", "VOST-FR", "VF", "VOF", "Frz. Live", "VO", "ST mal"]
elif language == "pl":
language = ["POL", "VOST-POL"]
if stream_language in language:
if stream_type in ("hls", "mp4"):
if urlparse(stream_url).path.endswith("m3u8"):
try:
streams = HLSStream.parse_variant_playlist(self.session, stream_url)
for stream in streams.items():
yield stream
except IOError as err:
self.logger.error("Failed to extract HLS streams: {0}", err)
else:
yield stream_name, HTTPStream(self.session, stream_url)
elif stream_type == "f4m":
try:
streams = HDSStream.parse_manifest(self.session, stream_url)
for stream in streams.items():
yield stream
except IOError as err:
self.logger.error("Failed to extract HDS streams: {0}", err)
def _get_streams(self):
match = self._url_re.match(self.url)
video_id = match.group('video_id')
if video_id is not None:
# VOD
res = self.session.http.get(self.VOD_API_URL.format(video_id))
stream_data = self.session.http.json(res, schema=self._vod_api_schema)
# Check whether video format is expired
current_date = datetime.strptime(stream_data['current_date'], '%Y-%m-%dT%H:%M:%S.%f')
valid_start = datetime.strptime(stream_data['shows']['valid_start'], '%Y-%m-%dT%H:%M:%S')
valid_end = datetime.strptime(stream_data['shows']['valid_end'], '%Y-%m-%dT%H:%M:%S')
if current_date < valid_start or current_date > valid_end:
self.logger.error('Failed to access stream, may be due to expired content')
return
streams = stream_data['shows']['media:group']
else:
# Live
channel_id = match.group('channel_id')
# Get live streams for desktop
res = self.session.http.get(self.LIVE_API_URL, params={'channelid': channel_id})
streams = self.session.http.xml(res, schema=self._live_api_schema)
# Get HLS streams for Iphone
res = self.session.http.get(self.LIVE_API_URL, params={'channelid': channel_id, 'platform': 'iphone'})
stream = self.session.http.json(res, schema=self._live_api_iphone_schema)
if stream != 'none':
streams.append(stream)
for stream in streams:
if '.f4m' in stream:
for s in HDSStream.parse_manifest(self.session, stream).items():
yield s
if '.m3u8' in stream:
for s in HLSStream.parse_variant_playlist(self.session, stream).items():
yield s
def _get_streams(self):
match = self._player_embed_re.match(self.url)
if match:
channel_id = match.group("channel_id")
else:
try:
res = http.get(self.url)
except Exception as e:
raise NoStreamsError(self.url)
channel_id = self._find_channel_id(res.text)
if not channel_id:
return
self.logger.info("Channel ID: {0}".format(channel_id))
res = http.get(self.PLAYER_EMBED_URL.format(channel_id))
items = self._find_stream_id(res.text)
if not items:
return
a = b = False
for stream in items:
if stream["abr"] == "hls":
try:
for s in HLSStream.parse_variant_playlist(self.session, stream["streamUrl"]).items():
yield s
except IOError:
a = True
elif stream["abr"] == "hds":
try:
for s in HDSStream.parse_manifest(self.session, stream["streamUrl"]).items():
yield s
except IOError:
b = True
if a and b:
self.logger.warning("Could not open the stream, perhaps the channel is offline")
def _get_hds_streams(self, info):
# Needs the hdcore parameter added
url = info["_stream"] + HDCORE_PARAMETER
return HDSStream.parse_manifest(self.session, url, pvswf=SWF_URL).items()
def _create_streams(self, type_, video_id):
url = self._generate_security_url(type_, video_id)
res = http.get(url)
return HDSStream.parse_manifest(self.session, res.text, cookies=res.cookies)
def _get_streams(self):
# Retrieve geolocation data
res = http.get(self.GEO_URL)
geo = http.json(res, schema=self._geo_schema)
country_code = geo['reponse']['geo_info']['country_code']
# Retrieve URL page and search for video ID
res = http.get(self.url)
if 'france.tv' in self.url:
match = self._pluzz_video_id_re.search(res.text)
elif 'ludo.fr' in self.url or 'zouzous.fr' in self.url:
match = self._jeunesse_video_id_re.search(res.text)
elif 'france3-regions.francetvinfo.fr' in self.url:
match = self._f3_regions_video_id_re.search(res.text)
elif 'sport.francetvinfo.fr' in self.url:
match = self._sport_video_id_re.search(res.text)
if match is None:
return
video_id = match.group('video_id')
# Retrieve SWF player URL
swf_url = None
res = http.get(self.PLAYER_GENERATOR_URL)
player_url = update_scheme(self.url, http.json(res, schema=self._player_schema)['result'])
res = http.get(player_url)
match = self._swf_re.search(res.text)
if match is not None:
swf_url = update_scheme(self.url, match.group(0))
res = http.get(self.API_URL.format(video_id))
videos = http.json(res, schema=self._api_schema)
now = time.time()
offline = False
geolocked = False
drm = False
expired = False
streams = []
for video in videos['videos']:
video_url = video['url']
# Check whether video format is available
if video['statut'] != 'ONLINE':
offline = offline or True
continue
# Check whether video format is geo-locked
if video['geoblocage'] is not None and country_code not in video['geoblocage']:
geolocked = geolocked or True
continue
# Check whether video is DRM-protected
if video['drm']:
drm = drm or True
continue
# Check whether video format is expired
available = False
for interval in video['plages_ouverture']:
available = (interval['debut'] or 0) <= now <= (interval['fin'] or sys.maxsize)
if available:
break
if not available:
expired = expired or True
continue
# TODO: add DASH streams once supported
if '.mpd' in video_url:
continue
if '.f4m' in video_url or 'france.tv' in self.url:
res = http.get(self.TOKEN_URL.format(video_url))
video_url = res.text
if '.f4m' in video_url and swf_url is not None:
for bitrate, stream in HDSStream.parse_manifest(self.session,
video_url,
is_akamai=True,
pvswf=swf_url).items():
# HDS videos with data in their manifest fragment token
# doesn't seem to be supported by HDSStream. Ignore such
# stream (but HDS stream having only the hdntl parameter in
# their manifest token will be provided)
pvtoken = stream.request_params['params'].get('pvtoken', '')
match = self._hds_pv_data_re.search(pvtoken)
if match is None:
streams.append((bitrate, stream))
elif '.m3u8' in video_url:
for stream in HLSStream.parse_variant_playlist(self.session, video_url).items():
streams.append(stream)
# HBB TV streams are not provided anymore by France Televisions
elif '.mp4' in video_url and '/hbbtv/' not in video_url:
match = self._mp4_bitrate_re.match(video_url)
if match is not None:
bitrate = match.group('bitrate')
else:
# Fallback bitrate (seems all France Televisions MP4 videos
# seem have such bitrate)
bitrate = '1500k'
streams.append((bitrate, HTTPStream(self.session, video_url)))
if self.get_option("mux_subtitles") and videos['subtitles'] != []:
substreams = {}
for subtitle in videos['subtitles']:
# TTML subtitles are available but not supported by FFmpeg
if subtitle['format'] == 'ttml':
continue
substreams[subtitle['type']] = HTTPStream(self.session, subtitle['url'])
for quality, stream in streams:
yield quality, MuxedStream(self.session, stream, subtitles=substreams)
else:
for stream in streams:
yield stream
if offline:
self.logger.error('Failed to access stream, may be due to offline content')
if geolocked:
self.logger.error('Failed to access stream, may be due to geo-restricted content')
if drm:
self.logger.error('Failed to access stream, may be due to DRM-protected content')
if expired:
self.logger.error('Failed to access stream, may be due to expired content')
def _get_streams(self):
url_match = _url_re.match(self.url)
if url_match:
if url_match.group(1) in _channel:
return HDSStream.parse_manifest(self.session, STREAM_INFO_URL.format(channel=_channel[url_match.group(1)]))
def _get_streams(self):
"""
Find all the streams for the ITV url
:return: Mapping of quality to stream
"""
soap_message = self._soap_request()
headers = {'Content-Length': '{0:d}'.format(len(soap_message)),
'Content-Type': 'text/xml; charset=utf-8',
'Host': 'secure-mercury.itv.com',
'Origin': 'http://www.itv.com',
'Referer': 'http://www.itv.com/Mercury/Mercury_VideoPlayer.swf?v=null',
'SOAPAction': "http://tempuri.org/PlaylistService/GetPlaylist",
'User-Agent': ITV_PLAYER_USER_AGENT,
"X-Requested-With": "ShockwaveFlash/16.0.0.305"}
res = http.post("https://secure-mercury.itv.com/PlaylistService.svc?wsdl",
headers=headers,
data=soap_message)
# Parse XML
xmldoc = http.xml(res)
# Check that geo region has been accepted
faultcode = xmldoc.find('.//faultcode')
if faultcode is not None:
if 'InvalidGeoRegion' in faultcode.text:
self.logger.error('Failed to retrieve playlist data '
'(invalid geo region)')
return None
# Look for <MediaFiles> tag (RTMP streams)
mediafiles = xmldoc.find('.//VideoEntries//MediaFiles')
# Look for <ManifestFile> tag (HDS streams)
manifestfile = xmldoc.find('.//VideoEntries//ManifestFile')
# No streams
if not mediafiles and not manifestfile:
return None
streams = {}
# Proxy not needed for media retrieval (Note: probably better to use flag)
# for x in ('http', 'https'):
# if x in http.proxies:
# http.proxies.pop(x);
# Parse RTMP streams
if mediafiles:
rtmp = mediafiles.attrib['base']
for mediafile in mediafiles.findall("MediaFile"):
playpath = mediafile.find("URL").text
rtmp_url = urlparse(rtmp)
app = (rtmp_url.path[1:] + '?' + rtmp_url.query).rstrip('?')
live = app == "live"
params = dict(rtmp="{u.scheme}://{u.netloc}{u.path}".format(u=rtmp_url),
app=app.rstrip('?'),
playpath=playpath,
swfVfy=LIVE_SWF_URL if live else ONDEMAND_SWF_URL,
timeout=10)
if live:
params['live'] = True
bitrate = int(mediafile.attrib['bitrate']) / 1000
quality = "{0:d}k".format(int(bitrate))
streams[quality] = RTMPStream(self.session, params)
# Parse HDS streams
if manifestfile:
url = manifestfile.find('URL').text
if urlparse(url).path.endswith('f4m'):
streams.update(
HDSStream.parse_manifest(self.session, url, pvswf=LIVE_SWF_URL)
)
return streams
