def metadata_channel(self, channel):
queries = [
self._gql_persisted_query(
"ChannelShell",
"c3ea5a669ec074a58df5c11ce3c27093fa38534c94286dc14b68a25d5adcbf55",
login=channel,
lcpVideosEnabled=False),
self._gql_persisted_query(
"StreamMetadata",
"059c4653b788f5bdb2f5a2d2a24b0ddc3831a15079001a3d927556a96fb0517f",
channelLogin=channel)
]
return self.call(
queries,
schema=validate.Schema(
[
validate.all(
{"data": {
"userOrError": {
"displayName": str
}
}}),
validate.all({
"data": {
"user": {
"lastBroadcast": {
"title": str
},
"stream": {
"game": {
"name": str
}
}
}
}
})
],
validate.union_get(
(0, "data", "userOrError", "displayName"),
(1, "data", "user", "lastBroadcast", "title"),
(1, "data", "user", "stream", "game", "name"))))
def _get_streams(self):
try:
data = self.session.http.get(self.url, schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//video[@id='brightcove_video_player']"),
validate.union_get("data-video-id", "data-account", "data-ad-config-id", "data-player")
))
except PluginError:
return
data_video_id, data_account, data_ad_config_id, data_player = data
url = self._PLAYER_URL.format(data_account=data_account, data_player=data_player)
policy_key = self.session.http.get(url, schema=validate.Schema(
validate.transform(self._policy_key_re.search),
validate.any(None, validate.get(1))
))
if not policy_key:
return
url = self._API_URL.format(data_account=data_account, data_video_id=data_video_id)
if data_ad_config_id is not None:
url = update_qsd(url, dict(ad_config_id=data_ad_config_id))
streams = self.session.http.get(
url,
headers={"Accept": "application/json;pk={0}".format(policy_key)},
schema=validate.Schema(
validate.parse_json(),
{
"sources": [{
validate.optional("type"): validate.text,
"src": validate.url(),
}],
},
validate.get("sources"),
validate.filter(lambda source: source.get("type") == "application/x-mpegURL")
)
)
for stream in streams:
return HLSStream.parse_variant_playlist(self.session, stream["src"])
def _schema_videodetails(cls, data):
schema = validate.Schema(
{
"videoDetails": {
"videoId": str,
"author": str,
"title": str,
validate.optional("isLive"): validate.transform(bool),
validate.optional("isLiveContent"): validate.transform(bool),
validate.optional("isLiveDvrEnabled"): validate.transform(bool),
validate.optional("isLowLatencyLiveStream"): validate.transform(bool),
validate.optional("isPrivate"): validate.transform(bool),
},
"microformat": validate.all(
validate.any(
validate.all(
{"playerMicroformatRenderer": dict},
validate.get("playerMicroformatRenderer")
),
validate.all(
{"microformatDataRenderer": dict},
validate.get("microformatDataRenderer")
)
),
{
"category": str
}
)
},
validate.union_get(
("videoDetails", "videoId"),
("videoDetails", "author"),
("microformat", "category"),
("videoDetails", "title"),
("videoDetails", "isLive")
)
)
videoDetails = validate.validate(schema, data)
log.trace(f"videoDetails = {videoDetails!r}")
return videoDetails
def metadata_clips(self, clipname):
queries = [
self._gql_persisted_query(
"ClipsView",
"4480c1dcc2494a17bb6ef64b94a5213a956afb8a45fe314c66b0d04079a93a8f",
slug=clipname),
self._gql_persisted_query(
"ClipsTitle",
"f6cca7f2fdfbfc2cecea0c88452500dae569191e58a265f97711f8f2a838f5b4",
slug=clipname)
]
return self.call(
queries,
schema=validate.Schema([
validate.all(
{
"data": {
"clip": {
"id": validate.text,
"broadcaster": {
"displayName": validate.text
},
"game": {
"name": validate.text
}
}
}
}, validate.get(("data", "clip"))),
validate.all({"data": {
"clip": {
"title": validate.text
}
}}, validate.get(("data", "clip")))
],
validate.union_get(
(0, "id"),
(0, "broadcaster", "displayName"),
(0, "game", "name"), (1, "title"))))
def _streams_brightcove_js(self, root):
re_js_src = re.compile(r"^[\w/]+/main\.\w+\.js$")
re_js_brightcove_video = re.compile(
r'i\?\([A-Z]="[^"]+",y="(?P<video_id>[0-9]+).*"data-account"\s*:\s*"(?P<account_id>[0-9]+)',
)
schema_brightcove_js = validate.Schema(
validate.xml_findall(r".//script[@src]"),
validate.filter(
lambda elem: re_js_src.search(elem.attrib.get("src"))),
validate.get(0), str,
validate.transform(lambda src: urljoin(self.url, src)))
schema_brightcove_js2 = validate.Schema(
validate.transform(re_js_brightcove_video.search),
validate.union_get("account_id", "video_id"))
try:
js_url = schema_brightcove_js.validate(root)
log.debug(f"JS URL: {js_url}")
account_id, video_id = self.session.http.get(
js_url, schema=schema_brightcove_js2)
except (PluginError, TypeError):
return
return self._brightcove(account_id, video_id)
def get_wss_api_url(self):
try:
data = self.session.http.get(self.url, schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//script[@id='embedded-data'][@data-props]"),
validate.get("data-props"),
validate.parse_json(),
{"site": {
"relive": {
"webSocketUrl": validate.url(scheme="wss")
},
validate.optional("frontendId"): int
}},
validate.get("site"),
validate.union_get(("relive", "webSocketUrl"), "frontendId")
))
except PluginError:
return
wss_api_url, frontend_id = data
if frontend_id is not None:
wss_api_url = update_qsd(wss_api_url, {"frontend_id": frontend_id})
return wss_api_url
def _get_streams(self):
zdf_json = self.session.http.get(self.url, schema=validate.Schema(
validate.transform(self._re_api_json.search),
validate.any(None, validate.all(
validate.get("json"),
validate.transform(parse_json),
{
"apiToken": str,
"content": validate.url()
},
validate.union_get("apiToken", "content")
))
))
if zdf_json is None:
return
apiToken, apiUrl = zdf_json
headers = {
"Accept": "application/vnd.de.zdf.v1.0+json;charset=UTF-8",
"Api-Auth": f"Bearer {apiToken}",
"Referer": self.url
}
pApiUrl = urlparse(apiUrl)
apiUrlBase = urlunparse((pApiUrl.scheme, pApiUrl.netloc, "", "", "", ""))
apiUrlPath = self.session.http.get(apiUrl, headers=headers, schema=validate.Schema(
validate.transform(parse_json),
{"mainVideoContent": {
"http://zdf.de/rels/target": {
"http://zdf.de/rels/streams/ptmd-template": str
}
}},
validate.get(("mainVideoContent", "http://zdf.de/rels/target", "http://zdf.de/rels/streams/ptmd-template")),
validate.transform(lambda template: template.format(playerId=self.PLAYER_ID).replace(" ", ""))
))
stream_request_url = url_concat(apiUrlBase, apiUrlPath)
data = self.session.http.get(stream_request_url, headers=headers, schema=validate.Schema(
validate.transform(parse_json),
{"priorityList": [{
"formitaeten": validate.all(
[{
"type": str,
"qualities": validate.all(
[{
"quality": str,
"audio": {
"tracks": [{
"uri": validate.url()
}]
}
}],
validate.filter(lambda obj: obj["quality"] == "auto")
)
}],
validate.filter(lambda obj: obj["type"] == "h264_aac_ts_http_m3u8_http")
)
}]},
validate.get("priorityList")
))
for priority in data:
for formitaeten in priority["formitaeten"]:
for quality in formitaeten["qualities"]:
for audio in quality["audio"]["tracks"]:
yield from HLSStream.parse_variant_playlist(self.session, audio["uri"], headers=headers).items()
def _get_streams(self):
res = self.session.http.get(self.url)
for script in itertags(res.text, "script"):
_type = script.attributes.get("type")
if not (_type and _type == "application/json"):
continue
video_url = None
_data_ssr_name = script.attributes.get("data-ssr-name")
if not _data_ssr_name:
continue
log.trace(f"Found _data_ssr_name={_data_ssr_name}")
if _data_ssr_name == "pages/Broadcasts/Broadcasts":
self.title, video_url, is_live = parse_json(
script.text,
schema=validate.Schema(
{
"currentLivestream": {
"is_live": bool,
"title": str,
"stream": validate.url(),
}
}, validate.get("currentLivestream"),
validate.union_get("title", "stream", "is_live")))
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name == "pages/Livestream/Livestream":
self.title, video_url, is_live = parse_json(
script.text,
schema=validate.Schema(
{
"streamIsLive": bool,
"title": str,
"stream": validate.url(),
}, validate.union_get("title", "stream",
"streamIsLive")))
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name in self.vod_keys.keys():
_key = self.vod_keys[_data_ssr_name]
self.title, video_url = parse_json(
script.text,
schema=validate.Schema(
{
_key: {
"title": str,
"aspect_ratios": {
"profiles":
validate.all(
[{
"name": str,
"url": validate.url(),
}],
validate.filter(lambda n: n["name"] ==
"hls_unencrypted"))
}
}
}, validate.get(_key),
validate.union_get(
"title", ("aspect_ratios", "profiles", 0, "url"))))
if video_url is not None:
yield from HLSStream.parse_variant_playlist(
self.session, video_url).items()
break
def _get_streams(self):
self.session.http.headers.update({
"User-Agent": useragents.CHROME
})
CHROME_VERSION = re.compile(r"Chrome/(\d+)").search(useragents.CHROME).group(1)
# Retrieve geolocation data
country_code = self.session.http.get(self.GEO_URL, schema=validate.Schema(
validate.parse_json(),
{"reponse": {"geo_info": {
"country_code": validate.text
}}},
validate.get(("reponse", "geo_info", "country_code"))
))
log.debug("Country: {0}".format(country_code))
# Retrieve URL page and search for video ID
video_id = None
try:
video_id = self.session.http.get(self.url, schema=validate.Schema(
validate.parse_html(),
validate.any(
validate.all(
validate.xml_xpath_string(".//script[contains(text(),'window.FTVPlayerVideos')][1]/text()"),
validate.text,
validate.transform(self._re_ftv_player_videos.search),
validate.get("json"),
validate.parse_json(),
[{"videoId": validate.text}],
validate.get((0, "videoId"))
),
validate.all(
validate.xml_xpath_string(".//script[contains(text(),'new Magnetoscope')][1]/text()"),
validate.text,
validate.transform(self._re_player_load.search),
validate.get("video_id")
),
validate.all(
validate.xml_xpath_string(".//*[@id][contains(@class,'francetv-player-wrapper')][1]/@id"),
validate.text
),
validate.all(
validate.xml_xpath_string(".//*[@data-id][@class='magneto'][1]/@data-id"),
validate.text
)
)
))
except PluginError:
pass
if not video_id:
return
log.debug("Video ID: {0}".format(video_id))
api_url = update_qsd(self.API_URL.format(video_id=video_id), {
"country_code": country_code,
"w": 1920,
"h": 1080,
"player_version": self.PLAYER_VERSION,
"domain": urlparse(self.url).netloc,
"device_type": "mobile",
"browser": "chrome",
"browser_version": CHROME_VERSION,
"os": "ios",
"gmt": datetime.now(tz=LOCALTIMEZONE).strftime("%z")
})
video_format, token_url, url, self.title = self.session.http.get(api_url, schema=validate.Schema(
validate.parse_json(),
{
"video": {
"workflow": validate.any("token-akamai", "dai"),
"format": validate.any("dash", "hls"),
"token": validate.url(),
"url": validate.url()
},
"meta": {
"title": validate.text
}
},
validate.union_get(
("video", "format"),
("video", "token"),
("video", "url"),
("meta", "title")
)
))
data_url = update_qsd(token_url, {
"url": url
})
video_url = self.session.http.get(data_url, schema=validate.Schema(
validate.parse_json(),
{"url": validate.url()},
validate.get("url")
))
if video_format == "dash":
for s in DASHStream.parse_manifest(self.session, video_url).items():
yield s
elif video_format == "hls":
for s in HLSStream.parse_variant_playlist(self.session, video_url).items():
yield s
def _get_streams(self):
try:
scripts = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_findall(
".//script[@type='application/json'][@data-ssr-name]"),
[
validate.union((validate.get("data-ssr-name"),
validate.all(validate.getattr("text"),
validate.parse_json())))
]))
except PluginError:
log.error("Could not find any stream data")
return
for _data_ssr_name, _data_json in scripts:
video_url = None
log.trace(f"Found _data_ssr_name={_data_ssr_name}")
if _data_ssr_name == "pages/Broadcasts/Broadcasts":
self.title, video_url, is_live = validate.Schema(
{
"currentLivestream": {
"is_live": bool,
"title": str,
"stream": validate.url(),
}
}, validate.get("currentLivestream"),
validate.union_get("title", "stream",
"is_live")).validate(_data_json)
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name == "pages/Livestream/Livestream":
self.title, video_url, is_live = validate.Schema(
{
"streamIsLive": bool,
"title": str,
"stream": validate.url(),
}, validate.union_get("title", "stream",
"streamIsLive")).validate(_data_json)
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name in self.vod_keys.keys():
_key = self.vod_keys[_data_ssr_name]
self.title, video_url = validate.Schema(
{
_key: {
"title": str,
"aspect_ratios": {
"profiles":
validate.all([{
"name": str,
"url": validate.url(),
}],
validate.filter(lambda p: p[
"name"] == "hls_unencrypted"))
}
}
}, validate.get(_key),
validate.union_get("title",
("aspect_ratios", "profiles", 0,
"url"))).validate(_data_json)
if video_url is not None:
return HLSStream.parse_variant_playlist(
self.session, video_url)
def clips(self, clipname):
queries = [
self._gql_persisted_query(
"VideoAccessToken_Clip",
"36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11",
slug=clipname),
self._gql_persisted_query(
"ClipsView",
"4480c1dcc2494a17bb6ef64b94a5213a956afb8a45fe314c66b0d04079a93a8f",
slug=clipname),
self._gql_persisted_query(
"ClipsTitle",
"f6cca7f2fdfbfc2cecea0c88452500dae569191e58a265f97711f8f2a838f5b4",
slug=clipname)
]
return self.call(
queries,
schema=validate.Schema([
validate.all(
{
"data": {
"clip": {
"playbackAccessToken":
validate.all({
"signature": str,
"value": str
}, validate.union_get("signature", "value")),
"videoQualities": [
validate.all(
{
"frameRate":
validate.transform(int),
"quality": str,
"sourceURL": validate.url()
},
validate.transform(lambda q: (
f"{q['quality']}p{q['frameRate']}",
q["sourceURL"])))
]
}
}
}, validate.get(("data", "clip")),
validate.union_get("playbackAccessToken",
"videoQualities")),
validate.all(
{
"data": {
"clip": {
"broadcaster": {
"displayName": str
},
"game": {
"name": str
}
}
}
}, validate.get(("data", "clip")),
validate.union_get(("broadcaster", "displayName"),
("game", "name"))),
validate.all({"data": {
"clip": {
"title": str
}
}}, validate.get(("data", "clip", "title")))
]))
def _get_streams(self):
self.session.http.headers.update({"Referer": self.url})
schema_data = validate.Schema(
validate.parse_html(),
validate.xml_xpath_string(
".//script[@id='__NEXT_DATA__'][text()]/text()"),
str,
validate.parse_json(),
{
"props": {
"pageProps": {
"data": {
"liveBroadcast": {
# "id": str,
"current":
validate.any(
None, {
"channel": str,
"channelName": str,
"legacyEncoder": str,
}),
"next":
validate.any(
None, {
"channel": str,
"channelName": str,
"legacyEncoder": str,
})
}
}
}
}
},
validate.get(("props", "pageProps", "data", "liveBroadcast")),
validate.union_get("current", "next"),
)
try:
data_current, data_next = self.session.http.get(self.url,
schema=schema_data)
except PluginError:
return
log.debug(f"current={data_current!r}")
log.debug(f"next={data_next!r}")
data = data_current or data_next
video_id = data["legacyEncoder"]
self.title = data["channelName"]
_hash = self.session.http.get(
"https://www.ceskatelevize.cz/v-api/iframe-hash/",
schema=validate.Schema(str))
res = self.session.http.get(
"https://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php",
params={
"hash": _hash,
"origin": "iVysilani",
"autoStart": "true",
"videoID": video_id,
},
)
m = self._re_playlist_info.search(res.text)
if not m:
return
_type, _id = m.groups()
data = self.session.http.post(
"https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/",
data={
"playlist[0][type]": _type,
"playlist[0][id]": _id,
"requestUrl": "/ivysilani/embed/iFramePlayer.php",
"requestSource": "iVysilani",
"type": "html",
"canPlayDRM": "false",
},
headers={
"x-addr": "127.0.0.1",
},
schema=validate.Schema(
validate.parse_json(), {
validate.optional("streamingProtocol"): str,
"url": validate.any(validate.url(), "Error",
"error_region")
}),
)
if data["url"] in ["Error", "error_region"]:
log.error("This stream is not available")
return
url = self.session.http.get(
data["url"],
schema=validate.Schema(
validate.parse_json(), {
"playlist": [{
validate.optional("type"): str,
"streamUrls": {
"main": validate.url(),
}
}]
}, validate.get(("playlist", 0, "streamUrls", "main"))))
return DASHStream.parse_manifest(self.session, url)
def _get_streams(self):
data_json = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_findtext(
".//script[@id='fetchedContextValue'][@type='application/json']"
),
validate.any(
None,
validate.all(
validate.parse_json(), {str: dict},
validate.transform(lambda obj: list(obj.items())),
validate.filter(lambda item: item[0].startswith(
"https://api.ardmediathek.de/page-gateway/pages/")
),
validate.any(validate.get((0, 1)), [])))))
if not data_json:
return
schema_data = validate.Schema({
"id":
str,
"widgets":
validate.all(
[dict],
validate.filter(lambda item: item.get("mediaCollection")),
validate.get(0),
validate.any(
None,
validate.all(
{
"geoblocked":
bool,
"publicationService": {
"name": str,
},
"show":
validate.any(
None,
validate.all({"title": str},
validate.get("title"))),
"title":
str,
"mediaCollection": {
"embedded": {
"_mediaArray": [
validate.all(
{
"_mediaStreamArray": [
validate.all(
{
"_quality":
validate.any(
str, int),
"_stream":
validate.url(),
},
validate.union_get(
"_quality",
"_stream"))
]
},
validate.get("_mediaStreamArray"),
validate.transform(dict))
]
}
},
},
validate.union_get(
"geoblocked",
("mediaCollection", "embedded", "_mediaArray", 0),
("publicationService", "name"),
"title",
"show",
))))
})
data = schema_data.validate(data_json)
log.debug(f"Found media id: {data['id']}")
if not data["widgets"]:
log.info("The content is unavailable")
return
geoblocked, media, self.author, self.title, show = data["widgets"]
if geoblocked:
log.info("The content is not available in your region")
return
if show:
self.title = f"{show}: {self.title}"
if media.get("auto"):
yield from HLSStream.parse_variant_playlist(
self.session, media.get("auto")).items()
else:
for quality, stream in media.items():
yield self._QUALITY_MAP.get(quality, quality), HTTPStream(
self.session, stream)
def _get_streams(self):
schema_metadata = validate.Schema(
validate.parse_json(), {
validate.optional("author"):
validate.all({"name": str}, validate.get("name")),
validate.optional("movie"):
validate.all({"title": str}, validate.get("title")),
validate.optional("hlsManifestUrl"):
validate.url(),
validate.optional("hlsMasterPlaylistUrl"):
validate.url(),
validate.optional("liveDashManifestUrl"):
validate.url(),
validate.optional("videos"): [
validate.all({
"name": str,
"url": validate.url()
}, validate.union_get("name", "url"))
]
})
try:
metadata, metadata_url = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//*[@data-options]"),
validate.get("data-options"), validate.parse_json(), {
"flashvars": {
validate.optional("metadata"):
str,
validate.optional("metadataUrl"):
validate.all(validate.transform(unquote),
validate.url())
}
}, validate.get("flashvars"),
validate.union_get("metadata", "metadataUrl")))
except PluginError:
log.error("Could not find metadata")
return
self.session.http.headers.update({"Referer": self.url})
if not metadata and metadata_url:
metadata = self.session.http.post(metadata_url).text
log.trace(f"{metadata!r}")
try:
data = schema_metadata.validate(metadata)
except PluginError:
log.error("Could not parse metadata")
return
self.author = data.get("author")
self.title = data.get("movie")
for hls_url in data.get("hlsManifestUrl"), data.get(
"hlsMasterPlaylistUrl"):
if hls_url is not None:
return HLSStream.parse_variant_playlist(self.session, hls_url)
if data.get("liveDashManifestUrl"):
return DASHStream.parse_manifest(self.session,
data.get("liveDashManifestUrl"))
return {
f"{self.QUALITY_WEIGHTS[name]}p" if name in self.QUALITY_WEIGHTS
else name: HTTPStream(self.session, url)
for name, url in data.get("videos", [])
}
def get_live(self, username):
netloc = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_xpath_string(
".//script[contains(@src,'/stream/player.js')][1]/@src"),
validate.any(
None,
validate.transform(lambda src: urlparse(src).netloc))))
if not netloc:
log.error("Could not find server netloc")
return
channel, multistreams = self.session.http.get(
self.API_URL_LIVE.format(username=username),
schema=validate.Schema(
validate.parse_json(), {
"channel":
validate.any(
None, {
"stream_name": validate.text,
"title": validate.text,
"online": bool,
"private": bool,
"categories": [{
"label": validate.text
}],
}),
"getMultiStreams":
validate.any(
None, {
"multistream": bool,
"streams": [{
"name": validate.text,
"online": bool,
}],
}),
}, validate.union_get("channel", "getMultiStreams")))
if not channel or not multistreams:
log.debug("Missing channel or streaming data")
return
log.trace("netloc={0!r}".format(netloc))
log.trace("channel={0!r}".format(channel))
log.trace("multistreams={0!r}".format(multistreams))
if not channel["online"]:
log.error("User is not online")
return
if channel["private"]:
log.info("This is a private stream")
return
self.author = username
self.category = channel["categories"][0]["label"]
self.title = channel["title"]
hls_url = self.HLS_URL.format(netloc=netloc,
file_name=channel["stream_name"])
return HLSStream.parse_variant_playlist(self.session, hls_url)
def test_union_get(self):
assert validate(union_get("foo", "bar"), {"foo": "alpha", "bar": "beta"}) == ("alpha", "beta")
assert validate(union_get("foo", "bar", seq=list), {"foo": "alpha", "bar": "beta"}) == ["alpha", "beta"]
assert validate(union_get(("foo", "bar"), ("baz", "qux")),
{"foo": {"bar": "alpha"}, "baz": {"qux": "beta"}}) == ("alpha", "beta")
