def metadata_channel(self, channel): queries = [ self._gql_persisted_query( "ChannelShell", "c3ea5a669ec074a58df5c11ce3c27093fa38534c94286dc14b68a25d5adcbf55", login=channel, lcpVideosEnabled=False), self._gql_persisted_query( "StreamMetadata", "059c4653b788f5bdb2f5a2d2a24b0ddc3831a15079001a3d927556a96fb0517f", channelLogin=channel) ] return self.call( queries, schema=validate.Schema( [ validate.all( {"data": { "userOrError": { "displayName": str } }}), validate.all({ "data": { "user": { "lastBroadcast": { "title": str }, "stream": { "game": { "name": str } } } } }) ], validate.union_get( (0, "data", "userOrError", "displayName"), (1, "data", "user", "lastBroadcast", "title"), (1, "data", "user", "stream", "game", "name"))))
def _get_streams(self): try: data = self.session.http.get(self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//video[@id='brightcove_video_player']"), validate.union_get("data-video-id", "data-account", "data-ad-config-id", "data-player") )) except PluginError: return data_video_id, data_account, data_ad_config_id, data_player = data url = self._PLAYER_URL.format(data_account=data_account, data_player=data_player) policy_key = self.session.http.get(url, schema=validate.Schema( validate.transform(self._policy_key_re.search), validate.any(None, validate.get(1)) )) if not policy_key: return url = self._API_URL.format(data_account=data_account, data_video_id=data_video_id) if data_ad_config_id is not None: url = update_qsd(url, dict(ad_config_id=data_ad_config_id)) streams = self.session.http.get( url, headers={"Accept": "application/json;pk={0}".format(policy_key)}, schema=validate.Schema( validate.parse_json(), { "sources": [{ validate.optional("type"): validate.text, "src": validate.url(), }], }, validate.get("sources"), validate.filter(lambda source: source.get("type") == "application/x-mpegURL") ) ) for stream in streams: return HLSStream.parse_variant_playlist(self.session, stream["src"])
def _schema_videodetails(cls, data): schema = validate.Schema( { "videoDetails": { "videoId": str, "author": str, "title": str, validate.optional("isLive"): validate.transform(bool), validate.optional("isLiveContent"): validate.transform(bool), validate.optional("isLiveDvrEnabled"): validate.transform(bool), validate.optional("isLowLatencyLiveStream"): validate.transform(bool), validate.optional("isPrivate"): validate.transform(bool), }, "microformat": validate.all( validate.any( validate.all( {"playerMicroformatRenderer": dict}, validate.get("playerMicroformatRenderer") ), validate.all( {"microformatDataRenderer": dict}, validate.get("microformatDataRenderer") ) ), { "category": str } ) }, validate.union_get( ("videoDetails", "videoId"), ("videoDetails", "author"), ("microformat", "category"), ("videoDetails", "title"), ("videoDetails", "isLive") ) ) videoDetails = validate.validate(schema, data) log.trace(f"videoDetails = {videoDetails!r}") return videoDetails
def metadata_clips(self, clipname): queries = [ self._gql_persisted_query( "ClipsView", "4480c1dcc2494a17bb6ef64b94a5213a956afb8a45fe314c66b0d04079a93a8f", slug=clipname), self._gql_persisted_query( "ClipsTitle", "f6cca7f2fdfbfc2cecea0c88452500dae569191e58a265f97711f8f2a838f5b4", slug=clipname) ] return self.call( queries, schema=validate.Schema([ validate.all( { "data": { "clip": { "id": validate.text, "broadcaster": { "displayName": validate.text }, "game": { "name": validate.text } } } }, validate.get(("data", "clip"))), validate.all({"data": { "clip": { "title": validate.text } }}, validate.get(("data", "clip"))) ], validate.union_get( (0, "id"), (0, "broadcaster", "displayName"), (0, "game", "name"), (1, "title"))))
def _streams_brightcove_js(self, root): re_js_src = re.compile(r"^[\w/]+/main\.\w+\.js$") re_js_brightcove_video = re.compile( r'i\?\([A-Z]="[^"]+",y="(?P<video_id>[0-9]+).*"data-account"\s*:\s*"(?P<account_id>[0-9]+)', ) schema_brightcove_js = validate.Schema( validate.xml_findall(r".//script[@src]"), validate.filter( lambda elem: re_js_src.search(elem.attrib.get("src"))), validate.get(0), str, validate.transform(lambda src: urljoin(self.url, src))) schema_brightcove_js2 = validate.Schema( validate.transform(re_js_brightcove_video.search), validate.union_get("account_id", "video_id")) try: js_url = schema_brightcove_js.validate(root) log.debug(f"JS URL: {js_url}") account_id, video_id = self.session.http.get( js_url, schema=schema_brightcove_js2) except (PluginError, TypeError): return return self._brightcove(account_id, video_id)
def get_wss_api_url(self): try: data = self.session.http.get(self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//script[@id='embedded-data'][@data-props]"), validate.get("data-props"), validate.parse_json(), {"site": { "relive": { "webSocketUrl": validate.url(scheme="wss") }, validate.optional("frontendId"): int }}, validate.get("site"), validate.union_get(("relive", "webSocketUrl"), "frontendId") )) except PluginError: return wss_api_url, frontend_id = data if frontend_id is not None: wss_api_url = update_qsd(wss_api_url, {"frontend_id": frontend_id}) return wss_api_url
def _get_streams(self): zdf_json = self.session.http.get(self.url, schema=validate.Schema( validate.transform(self._re_api_json.search), validate.any(None, validate.all( validate.get("json"), validate.transform(parse_json), { "apiToken": str, "content": validate.url() }, validate.union_get("apiToken", "content") )) )) if zdf_json is None: return apiToken, apiUrl = zdf_json headers = { "Accept": "application/vnd.de.zdf.v1.0+json;charset=UTF-8", "Api-Auth": f"Bearer {apiToken}", "Referer": self.url } pApiUrl = urlparse(apiUrl) apiUrlBase = urlunparse((pApiUrl.scheme, pApiUrl.netloc, "", "", "", "")) apiUrlPath = self.session.http.get(apiUrl, headers=headers, schema=validate.Schema( validate.transform(parse_json), {"mainVideoContent": { "http://zdf.de/rels/target": { "http://zdf.de/rels/streams/ptmd-template": str } }}, validate.get(("mainVideoContent", "http://zdf.de/rels/target", "http://zdf.de/rels/streams/ptmd-template")), validate.transform(lambda template: template.format(playerId=self.PLAYER_ID).replace(" ", "")) )) stream_request_url = url_concat(apiUrlBase, apiUrlPath) data = self.session.http.get(stream_request_url, headers=headers, schema=validate.Schema( validate.transform(parse_json), {"priorityList": [{ "formitaeten": validate.all( [{ "type": str, "qualities": validate.all( [{ "quality": str, "audio": { "tracks": [{ "uri": validate.url() }] } }], validate.filter(lambda obj: obj["quality"] == "auto") ) }], validate.filter(lambda obj: obj["type"] == "h264_aac_ts_http_m3u8_http") ) }]}, validate.get("priorityList") )) for priority in data: for formitaeten in priority["formitaeten"]: for quality in formitaeten["qualities"]: for audio in quality["audio"]["tracks"]: yield from HLSStream.parse_variant_playlist(self.session, audio["uri"], headers=headers).items()
def _get_streams(self): res = self.session.http.get(self.url) for script in itertags(res.text, "script"): _type = script.attributes.get("type") if not (_type and _type == "application/json"): continue video_url = None _data_ssr_name = script.attributes.get("data-ssr-name") if not _data_ssr_name: continue log.trace(f"Found _data_ssr_name={_data_ssr_name}") if _data_ssr_name == "pages/Broadcasts/Broadcasts": self.title, video_url, is_live = parse_json( script.text, schema=validate.Schema( { "currentLivestream": { "is_live": bool, "title": str, "stream": validate.url(), } }, validate.get("currentLivestream"), validate.union_get("title", "stream", "is_live"))) if not is_live: log.error(self._msg_live_offline) continue elif _data_ssr_name == "pages/Livestream/Livestream": self.title, video_url, is_live = parse_json( script.text, schema=validate.Schema( { "streamIsLive": bool, "title": str, "stream": validate.url(), }, validate.union_get("title", "stream", "streamIsLive"))) if not is_live: log.error(self._msg_live_offline) continue elif _data_ssr_name in self.vod_keys.keys(): _key = self.vod_keys[_data_ssr_name] self.title, video_url = parse_json( script.text, schema=validate.Schema( { _key: { "title": str, "aspect_ratios": { "profiles": validate.all( [{ "name": str, "url": validate.url(), }], validate.filter(lambda n: n["name"] == "hls_unencrypted")) } } }, validate.get(_key), validate.union_get( "title", ("aspect_ratios", "profiles", 0, "url")))) if video_url is not None: yield from HLSStream.parse_variant_playlist( self.session, video_url).items() break
def _get_streams(self): self.session.http.headers.update({ "User-Agent": useragents.CHROME }) CHROME_VERSION = re.compile(r"Chrome/(\d+)").search(useragents.CHROME).group(1) # Retrieve geolocation data country_code = self.session.http.get(self.GEO_URL, schema=validate.Schema( validate.parse_json(), {"reponse": {"geo_info": { "country_code": validate.text }}}, validate.get(("reponse", "geo_info", "country_code")) )) log.debug("Country: {0}".format(country_code)) # Retrieve URL page and search for video ID video_id = None try: video_id = self.session.http.get(self.url, schema=validate.Schema( validate.parse_html(), validate.any( validate.all( validate.xml_xpath_string(".//script[contains(text(),'window.FTVPlayerVideos')][1]/text()"), validate.text, validate.transform(self._re_ftv_player_videos.search), validate.get("json"), validate.parse_json(), [{"videoId": validate.text}], validate.get((0, "videoId")) ), validate.all( validate.xml_xpath_string(".//script[contains(text(),'new Magnetoscope')][1]/text()"), validate.text, validate.transform(self._re_player_load.search), validate.get("video_id") ), validate.all( validate.xml_xpath_string(".//*[@id][contains(@class,'francetv-player-wrapper')][1]/@id"), validate.text ), validate.all( validate.xml_xpath_string(".//*[@data-id][@class='magneto'][1]/@data-id"), validate.text ) ) )) except PluginError: pass if not video_id: return log.debug("Video ID: {0}".format(video_id)) api_url = update_qsd(self.API_URL.format(video_id=video_id), { "country_code": country_code, "w": 1920, "h": 1080, "player_version": self.PLAYER_VERSION, "domain": urlparse(self.url).netloc, "device_type": "mobile", "browser": "chrome", "browser_version": CHROME_VERSION, "os": "ios", "gmt": datetime.now(tz=LOCALTIMEZONE).strftime("%z") }) video_format, token_url, url, self.title = self.session.http.get(api_url, schema=validate.Schema( validate.parse_json(), { "video": { "workflow": validate.any("token-akamai", "dai"), "format": validate.any("dash", "hls"), "token": validate.url(), "url": validate.url() }, "meta": { "title": validate.text } }, validate.union_get( ("video", "format"), ("video", "token"), ("video", "url"), ("meta", "title") ) )) data_url = update_qsd(token_url, { "url": url }) video_url = self.session.http.get(data_url, schema=validate.Schema( validate.parse_json(), {"url": validate.url()}, validate.get("url") )) if video_format == "dash": for s in DASHStream.parse_manifest(self.session, video_url).items(): yield s elif video_format == "hls": for s in HLSStream.parse_variant_playlist(self.session, video_url).items(): yield s
def _get_streams(self): try: scripts = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_findall( ".//script[@type='application/json'][@data-ssr-name]"), [ validate.union((validate.get("data-ssr-name"), validate.all(validate.getattr("text"), validate.parse_json()))) ])) except PluginError: log.error("Could not find any stream data") return for _data_ssr_name, _data_json in scripts: video_url = None log.trace(f"Found _data_ssr_name={_data_ssr_name}") if _data_ssr_name == "pages/Broadcasts/Broadcasts": self.title, video_url, is_live = validate.Schema( { "currentLivestream": { "is_live": bool, "title": str, "stream": validate.url(), } }, validate.get("currentLivestream"), validate.union_get("title", "stream", "is_live")).validate(_data_json) if not is_live: log.error(self._msg_live_offline) continue elif _data_ssr_name == "pages/Livestream/Livestream": self.title, video_url, is_live = validate.Schema( { "streamIsLive": bool, "title": str, "stream": validate.url(), }, validate.union_get("title", "stream", "streamIsLive")).validate(_data_json) if not is_live: log.error(self._msg_live_offline) continue elif _data_ssr_name in self.vod_keys.keys(): _key = self.vod_keys[_data_ssr_name] self.title, video_url = validate.Schema( { _key: { "title": str, "aspect_ratios": { "profiles": validate.all([{ "name": str, "url": validate.url(), }], validate.filter(lambda p: p[ "name"] == "hls_unencrypted")) } } }, validate.get(_key), validate.union_get("title", ("aspect_ratios", "profiles", 0, "url"))).validate(_data_json) if video_url is not None: return HLSStream.parse_variant_playlist( self.session, video_url)
def clips(self, clipname): queries = [ self._gql_persisted_query( "VideoAccessToken_Clip", "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11", slug=clipname), self._gql_persisted_query( "ClipsView", "4480c1dcc2494a17bb6ef64b94a5213a956afb8a45fe314c66b0d04079a93a8f", slug=clipname), self._gql_persisted_query( "ClipsTitle", "f6cca7f2fdfbfc2cecea0c88452500dae569191e58a265f97711f8f2a838f5b4", slug=clipname) ] return self.call( queries, schema=validate.Schema([ validate.all( { "data": { "clip": { "playbackAccessToken": validate.all({ "signature": str, "value": str }, validate.union_get("signature", "value")), "videoQualities": [ validate.all( { "frameRate": validate.transform(int), "quality": str, "sourceURL": validate.url() }, validate.transform(lambda q: ( f"{q['quality']}p{q['frameRate']}", q["sourceURL"]))) ] } } }, validate.get(("data", "clip")), validate.union_get("playbackAccessToken", "videoQualities")), validate.all( { "data": { "clip": { "broadcaster": { "displayName": str }, "game": { "name": str } } } }, validate.get(("data", "clip")), validate.union_get(("broadcaster", "displayName"), ("game", "name"))), validate.all({"data": { "clip": { "title": str } }}, validate.get(("data", "clip", "title"))) ]))
def _get_streams(self): self.session.http.headers.update({"Referer": self.url}) schema_data = validate.Schema( validate.parse_html(), validate.xml_xpath_string( ".//script[@id='__NEXT_DATA__'][text()]/text()"), str, validate.parse_json(), { "props": { "pageProps": { "data": { "liveBroadcast": { # "id": str, "current": validate.any( None, { "channel": str, "channelName": str, "legacyEncoder": str, }), "next": validate.any( None, { "channel": str, "channelName": str, "legacyEncoder": str, }) } } } } }, validate.get(("props", "pageProps", "data", "liveBroadcast")), validate.union_get("current", "next"), ) try: data_current, data_next = self.session.http.get(self.url, schema=schema_data) except PluginError: return log.debug(f"current={data_current!r}") log.debug(f"next={data_next!r}") data = data_current or data_next video_id = data["legacyEncoder"] self.title = data["channelName"] _hash = self.session.http.get( "https://www.ceskatelevize.cz/v-api/iframe-hash/", schema=validate.Schema(str)) res = self.session.http.get( "https://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php", params={ "hash": _hash, "origin": "iVysilani", "autoStart": "true", "videoID": video_id, }, ) m = self._re_playlist_info.search(res.text) if not m: return _type, _id = m.groups() data = self.session.http.post( "https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/", data={ "playlist[0][type]": _type, "playlist[0][id]": _id, "requestUrl": "/ivysilani/embed/iFramePlayer.php", "requestSource": "iVysilani", "type": "html", "canPlayDRM": "false", }, headers={ "x-addr": "127.0.0.1", }, schema=validate.Schema( validate.parse_json(), { validate.optional("streamingProtocol"): str, "url": validate.any(validate.url(), "Error", "error_region") }), ) if data["url"] in ["Error", "error_region"]: log.error("This stream is not available") return url = self.session.http.get( data["url"], schema=validate.Schema( validate.parse_json(), { "playlist": [{ validate.optional("type"): str, "streamUrls": { "main": validate.url(), } }] }, validate.get(("playlist", 0, "streamUrls", "main")))) return DASHStream.parse_manifest(self.session, url)
def _get_streams(self): data_json = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_findtext( ".//script[@id='fetchedContextValue'][@type='application/json']" ), validate.any( None, validate.all( validate.parse_json(), {str: dict}, validate.transform(lambda obj: list(obj.items())), validate.filter(lambda item: item[0].startswith( "https://api.ardmediathek.de/page-gateway/pages/") ), validate.any(validate.get((0, 1)), []))))) if not data_json: return schema_data = validate.Schema({ "id": str, "widgets": validate.all( [dict], validate.filter(lambda item: item.get("mediaCollection")), validate.get(0), validate.any( None, validate.all( { "geoblocked": bool, "publicationService": { "name": str, }, "show": validate.any( None, validate.all({"title": str}, validate.get("title"))), "title": str, "mediaCollection": { "embedded": { "_mediaArray": [ validate.all( { "_mediaStreamArray": [ validate.all( { "_quality": validate.any( str, int), "_stream": validate.url(), }, validate.union_get( "_quality", "_stream")) ] }, validate.get("_mediaStreamArray"), validate.transform(dict)) ] } }, }, validate.union_get( "geoblocked", ("mediaCollection", "embedded", "_mediaArray", 0), ("publicationService", "name"), "title", "show", )))) }) data = schema_data.validate(data_json) log.debug(f"Found media id: {data['id']}") if not data["widgets"]: log.info("The content is unavailable") return geoblocked, media, self.author, self.title, show = data["widgets"] if geoblocked: log.info("The content is not available in your region") return if show: self.title = f"{show}: {self.title}" if media.get("auto"): yield from HLSStream.parse_variant_playlist( self.session, media.get("auto")).items() else: for quality, stream in media.items(): yield self._QUALITY_MAP.get(quality, quality), HTTPStream( self.session, stream)
def _get_streams(self): schema_metadata = validate.Schema( validate.parse_json(), { validate.optional("author"): validate.all({"name": str}, validate.get("name")), validate.optional("movie"): validate.all({"title": str}, validate.get("title")), validate.optional("hlsManifestUrl"): validate.url(), validate.optional("hlsMasterPlaylistUrl"): validate.url(), validate.optional("liveDashManifestUrl"): validate.url(), validate.optional("videos"): [ validate.all({ "name": str, "url": validate.url() }, validate.union_get("name", "url")) ] }) try: metadata, metadata_url = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//*[@data-options]"), validate.get("data-options"), validate.parse_json(), { "flashvars": { validate.optional("metadata"): str, validate.optional("metadataUrl"): validate.all(validate.transform(unquote), validate.url()) } }, validate.get("flashvars"), validate.union_get("metadata", "metadataUrl"))) except PluginError: log.error("Could not find metadata") return self.session.http.headers.update({"Referer": self.url}) if not metadata and metadata_url: metadata = self.session.http.post(metadata_url).text log.trace(f"{metadata!r}") try: data = schema_metadata.validate(metadata) except PluginError: log.error("Could not parse metadata") return self.author = data.get("author") self.title = data.get("movie") for hls_url in data.get("hlsManifestUrl"), data.get( "hlsMasterPlaylistUrl"): if hls_url is not None: return HLSStream.parse_variant_playlist(self.session, hls_url) if data.get("liveDashManifestUrl"): return DASHStream.parse_manifest(self.session, data.get("liveDashManifestUrl")) return { f"{self.QUALITY_WEIGHTS[name]}p" if name in self.QUALITY_WEIGHTS else name: HTTPStream(self.session, url) for name, url in data.get("videos", []) }
def get_live(self, username): netloc = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_xpath_string( ".//script[contains(@src,'/stream/player.js')][1]/@src"), validate.any( None, validate.transform(lambda src: urlparse(src).netloc)))) if not netloc: log.error("Could not find server netloc") return channel, multistreams = self.session.http.get( self.API_URL_LIVE.format(username=username), schema=validate.Schema( validate.parse_json(), { "channel": validate.any( None, { "stream_name": validate.text, "title": validate.text, "online": bool, "private": bool, "categories": [{ "label": validate.text }], }), "getMultiStreams": validate.any( None, { "multistream": bool, "streams": [{ "name": validate.text, "online": bool, }], }), }, validate.union_get("channel", "getMultiStreams"))) if not channel or not multistreams: log.debug("Missing channel or streaming data") return log.trace("netloc={0!r}".format(netloc)) log.trace("channel={0!r}".format(channel)) log.trace("multistreams={0!r}".format(multistreams)) if not channel["online"]: log.error("User is not online") return if channel["private"]: log.info("This is a private stream") return self.author = username self.category = channel["categories"][0]["label"] self.title = channel["title"] hls_url = self.HLS_URL.format(netloc=netloc, file_name=channel["stream_name"]) return HLSStream.parse_variant_playlist(self.session, hls_url)
def test_union_get(self): assert validate(union_get("foo", "bar"), {"foo": "alpha", "bar": "beta"}) == ("alpha", "beta") assert validate(union_get("foo", "bar", seq=list), {"foo": "alpha", "bar": "beta"}) == ["alpha", "beta"] assert validate(union_get(("foo", "bar"), ("baz", "qux")), {"foo": {"bar": "alpha"}, "baz": {"qux": "beta"}}) == ("alpha", "beta")