def test_xml_find(self): el = Element("parent") el.append(Element("foo")) el.append(Element("bar")) assert validate(xml_find("bar"), el).tag == "bar" with self.assertRaises(ValueError) as cm: validate(xml_find("baz"), el) assert str(cm.exception) == "XPath 'baz' did not return an element"
def _streams_brightcove(self, root): schema_brightcove = validate.Schema( validate.any( validate.all(validate.xml_find(".//*[@accountid][@videoid]"), validate.union_get("accountid", "videoid")), validate.all( validate.xml_find(".//*[@data-account][@data-video-id]"), validate.union_get("data-account", "data-video-id")))) try: account_id, video_id = schema_brightcove.validate(root) except PluginError: return return self._brightcove(account_id, video_id)
def get_wss_api_url(self): try: data = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find( ".//script[@id='embedded-data'][@data-props]"), validate.get("data-props"), validate.parse_json(), { "site": { "relive": { "webSocketUrl": validate.url(scheme="wss") }, validate.optional("frontendId"): int } }, validate.get("site"), validate.union_get(("relive", "webSocketUrl"), "frontendId"))) except PluginError: return wss_api_url, frontend_id = data if frontend_id is not None: wss_api_url = update_qsd(wss_api_url, {"frontend_id": frontend_id}) return wss_api_url
def test_failure_schema(self): with pytest.raises(validate.ValidationError) as cm: validate.validate(validate.xml_find("."), "not-an-element") assert_validationerror( cm.value, """ ValidationError(Callable): iselement('not-an-element') is not true """)
def test_failure_not_found(self): with pytest.raises(validate.ValidationError) as cm: validate.validate(validate.xml_find("invalid"), Element("foo")) assert_validationerror( cm.value, """ ValidationError(xml_find): XPath 'invalid' did not return an element """)
def _get_streams(self): try: data_url = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//*[@data-ctrl-player]"), validate.get("data-ctrl-player"), validate.transform(lambda s: s.replace("'", "\"")), validate.parse_json(), {"url": validate.text}, validate.get("url"))) except PluginError: return data_url = urljoin(self._URL_DATA_BASE, data_url) log.debug("Player URL: '{0}'", data_url) self.title, media = self.session.http.get( data_url, schema=validate.Schema( validate.parse_json(name="MEDIAINFO"), { "mc": { validate.optional("_title"): validate.text, "_mediaArray": [ validate.all( { "_mediaStreamArray": [ validate.all( { "_quality": validate.any( validate.text, int), "_stream": [validate.url()], }, validate.union_get( "_quality", ("_stream", 0))) ] }, validate.get("_mediaStreamArray"), validate.transform(dict)) ] } }, validate.get("mc"), validate.union_get("_title", ("_mediaArray", 0)))) if media.get("auto"): for s in HLSStream.parse_variant_playlist( self.session, media.get("auto")).items(): yield s else: for quality, stream in media.items(): yield self._QUALITY_MAP.get(quality, quality), HTTPStream( self.session, stream)
def _schema_consent(data): schema_consent = validate.Schema( validate.parse_html(), validate.any( validate.xml_find(".//form[@action='https://consent.youtube.com/s']"), validate.all( validate.xml_xpath(".//form[@action='https://consent.youtube.com/save']"), validate.filter(lambda elem: elem.xpath(".//input[@type='hidden'][@name='set_ytc'][@value='true']")), validate.get(0), ) ), validate.union(( validate.get("action"), validate.xml_xpath(".//input[@type='hidden']"), )), ) return schema_consent.validate(data)
def _get_streams(self): try: data = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find( ".//video[@id='brightcove_video_player']"), validate.union_get("data-video-id", "data-account", "data-ad-config-id", "data-player"))) except PluginError: return data_video_id, data_account, data_ad_config_id, data_player = data url = self._PLAYER_URL.format(data_account=data_account, data_player=data_player) policy_key = self.session.http.get( url, schema=validate.Schema( validate.transform(self._policy_key_re.search), validate.any(None, validate.get(1)))) if not policy_key: return url = self._API_URL.format(data_account=data_account, data_video_id=data_video_id) if data_ad_config_id is not None: url = update_qsd(url, dict(ad_config_id=data_ad_config_id)) streams = self.session.http.get( url, headers={"Accept": f"application/json;pk={policy_key}"}, schema=validate.Schema( validate.parse_json(), { "sources": [{ validate.optional("type"): str, "src": validate.url(), }], }, validate.get("sources"), validate.filter(lambda source: source.get("type") == "application/x-mpegURL"))) for stream in streams: return HLSStream.parse_variant_playlist(self.session, stream["src"])
def _get_streams(self): stream_id, has_token, hls_url, dash_url = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//*[@data-video-id]"), validate.union(( validate.get("data-video-id"), validate.all( validate.get("data-video-has-token"), validate.transform(lambda val: val and val != "false"), ), validate.get("data-vjs-clip-hls-url"), validate.get("data-vjs-clip-dash-url"), )), ), ) if dash_url and has_token: token = self._get_stream_token(stream_id, "dash") parsed = urlsplit(dash_url) dash_url = urlunsplit( parsed._replace(path=f"{token}{parsed.path}")) return DASHStream.parse_manifest( self.session, dash_url, headers={"Referer": self.url}, ) if not hls_url: return if has_token: token = self._get_stream_token(stream_id, "hls") hls_url = f"{hls_url}?{token}" return HLSStream.parse_variant_playlist( self.session, hls_url, headers={"Referer": self.url}, )
def test_xml_find(self): el = Element("parent") el.append(Element("foo")) el.append(Element("bar")) assert validate(xml_find("bar"), el).tag == "bar"
"bitrate": int, "height": int }], validate.optional("play_url"): validate.url(scheme="http"), validate.optional("m3u8_url"): validate.url( scheme="http", path=validate.endswith(".m3u8") ), }, None) }, validate.optional("playerUri"): validate.text }) _smil_schema = validate.Schema(validate.union({ "http_base": validate.all( validate.xml_find("{http://www.w3.org/2001/SMIL20/Language}head/" "{http://www.w3.org/2001/SMIL20/Language}meta" "[@name='httpBase']"), validate.xml_element(attrib={ "content": validate.text }), validate.get("content") ), "videos": validate.all( validate.xml_findall("{http://www.w3.org/2001/SMIL20/Language}body/" "{http://www.w3.org/2001/SMIL20/Language}switch/" "{http://www.w3.org/2001/SMIL20/Language}video"), [ validate.all( validate.xml_element(attrib={ "src": validate.text, "system-bitrate": validate.all(
def test_xml_find(self): el = Element("parent") el.append(Element("foo")) el.append(Element("bar")) assert validate(xml_find("bar"), el).tag == "bar"
def _get_streams(self): schema_metadata = validate.Schema( validate.parse_json(), { validate.optional("author"): validate.all({"name": str}, validate.get("name")), validate.optional("movie"): validate.all({"title": str}, validate.get("title")), validate.optional("hlsManifestUrl"): validate.url(), validate.optional("hlsMasterPlaylistUrl"): validate.url(), validate.optional("liveDashManifestUrl"): validate.url(), validate.optional("videos"): [ validate.all({ "name": str, "url": validate.url() }, validate.union_get("name", "url")) ] }) try: metadata, metadata_url = self.session.http.get( self.url, schema=validate.Schema( validate.parse_html(), validate.xml_find(".//*[@data-options]"), validate.get("data-options"), validate.parse_json(), { "flashvars": { validate.optional("metadata"): str, validate.optional("metadataUrl"): validate.all(validate.transform(unquote), validate.url()) } }, validate.get("flashvars"), validate.union_get("metadata", "metadataUrl"))) except PluginError: log.error("Could not find metadata") return self.session.http.headers.update({"Referer": self.url}) if not metadata and metadata_url: metadata = self.session.http.post(metadata_url).text log.trace(f"{metadata!r}") try: data = schema_metadata.validate(metadata) except PluginError: log.error("Could not parse metadata") return self.author = data.get("author") self.title = data.get("movie") for hls_url in data.get("hlsManifestUrl"), data.get( "hlsMasterPlaylistUrl"): if hls_url is not None: return HLSStream.parse_variant_playlist(self.session, hls_url) if data.get("liveDashManifestUrl"): return DASHStream.parse_manifest(self.session, data.get("liveDashManifestUrl")) return { f"{self.QUALITY_WEIGHTS[name]}p" if name in self.QUALITY_WEIGHTS else name: HTTPStream(self.session, url) for name, url in data.get("videos", []) }
_url_re = re.compile(r"http(s)?://(?:(\w+\.)?ardmediathek.de/tv|mediathek.daserste.de/)") _media_id_re = re.compile(r"/play/(?:media|config)/(\d+)") _media_schema = validate.Schema({ "_mediaArray": [{ "_mediaStreamArray": [{ validate.optional("_server"): validate.text, "_stream": validate.any(validate.text, [validate.text]), "_quality": validate.any(int, validate.text) }] }] }) _smil_schema = validate.Schema( validate.union({ "base": validate.all( validate.xml_find("head/meta"), validate.get("base"), validate.url(scheme="http") ), "cdn": validate.all( validate.xml_find("head/meta"), validate.get("cdn") ), "videos": validate.all( validate.xml_findall("body/seq/video"), [validate.get("src")] ) }) )
import re from streamlink.plugin import Plugin from streamlink.plugin.api import http, validate from streamlink.stream import RTMPStream _url_re = re.compile("http(s)?://(\w+.)?beam.pro/(?P<channel>[^/]+)") CHANNEL_INFO = "https://beam.pro/api/v1/channels/{0}" CHANNEL_MANIFEST = "https://beam.pro/api/v1/channels/{0}/manifest.smil" _assets_schema = validate.Schema( validate.union({ "base": validate.all( validate.xml_find("./head/meta"), validate.get("base"), validate.url(scheme="rtmp") ), "videos": validate.all( validate.xml_findall(".//video"), [ validate.union({ "src": validate.all( validate.get("src"), validate.text ), "height": validate.all( validate.get("height"), validate.text, validate.transform(int) )
def test_success(self): element = Element("foo") assert validate.validate(validate.xml_find("."), element) is element
_url_re = re.compile( r""" http(s)?://(\w+\.)?cybergame.tv (?: /videos/(?P<video_id>\d+) )? (?: /(?P<channel>[^/&?]+) )? """, re.VERBOSE) _playlist_schema = validate.Schema( validate.union({ "base": validate.all(validate.xml_find("./head/meta"), validate.get("base"), validate.url(scheme="rtmp")), "videos": validate.all(validate.xml_findall(".//video"), [ validate.union({ "src": validate.all(validate.get("src"), validate.text), "height": validate.all(validate.get("height"), validate.text, validate.transform(int)) }) ]) })) class Cybergame(Plugin):
class DeutscheWelle(Plugin): default_channel = "1" url_re = re.compile(r"https?://(?:www\.)?dw\.com/") channel_re = re.compile(r'''<a.*?data-id="(\d+)".*?class="ici"''') live_stream_div = re.compile( r''' <div\s+class="mediaItem"\s+data-channel-id="(\d+)".*?>.*? <input\s+type="hidden"\s+name="file_name"\s+value="(.*?)"\s*>.*?<div ''', re.DOTALL | re.VERBOSE) smil_api_url = "http://www.dw.com/smil/{}" html5_api_url = "http://www.dw.com/html5Resource/{}" vod_player_type_re = re.compile( r'<input type="hidden" name="player_type" value="(?P<stream_type>.+?)">' ) stream_vod_data_re = re.compile( r'<input\s+type="hidden"\s+name="file_name"\s+value="(?P<stream_url>.+?)">.*?' r'<input\s+type="hidden"\s+name="media_id"\s+value="(?P<stream_id>\d+)">', re.DOTALL) smil_schema = validate.Schema( validate.union({ "base": validate.all(validate.xml_find(".//meta"), validate.xml_element(attrib={"base": validate.text}), validate.get("base")), "streams": validate.all(validate.xml_findall(".//switch/*"), [ validate.all( validate.getattr("attrib"), { "src": validate.text, "system-bitrate": validate.all( validate.text, validate.transform(int), ), validate.optional("width"): validate.all(validate.text, validate.transform(int)) }) ]) })) @classmethod def can_handle_url(cls, url): return cls.url_re.match(url) is not None def _create_stream(self, url, quality=None): if url.startswith('rtmp://'): return (quality, RTMPStream(self.session, {'rtmp': url})) if url.endswith('.m3u8'): return HLSStream.parse_variant_playlist(self.session, url).items() return (quality, HTTPStream(self.session, url)) def _get_live_streams(self, page): # check if a different language has been selected qs = dict(parse_qsl(urlparse(self.url).query)) channel = qs.get("channel") if not channel: m = self.channel_re.search(page.text) channel = m and m.group(1) self.logger.debug("Using sub-channel ID: {0}", channel) # extract the streams from the page, mapping between channel-id and stream url media_items = self.live_stream_div.finditer(page.text) stream_map = dict([mi.groups((1, 2)) for mi in media_items]) stream_url = stream_map.get(str(channel) or self.default_channel) if stream_url: return self._create_stream(stream_url) def _get_vod_streams(self, stream_type, page): m = self.stream_vod_data_re.search(page.text) if m is None: return stream_url, stream_id = m.groups() if stream_type == "video": stream_api_id = "v-{}".format(stream_id) default_quality = "vod" elif stream_type == "audio": stream_api_id = "a-{}".format(stream_id) default_quality = "audio" else: return # Retrieve stream embedded in web page yield self._create_stream(stream_url, default_quality) # Retrieve streams using API res = self.session.http.get(self.smil_api_url.format(stream_api_id)) videos = self.session.http.xml(res, schema=self.smil_schema) for video in videos['streams']: url = videos["base"] + video["src"] if url == stream_url or url.replace("_dwdownload.", ".") == stream_url: continue if video["system-bitrate"] > 0: # If width is available, use it to select the best stream # amongst those with same bitrate quality = "{}k".format( (video["system-bitrate"] + video.get("width", 0)) // 1000) else: quality = default_quality yield self._create_stream(url, quality) def _get_streams(self): res = self.session.http.get(self.url) m = self.vod_player_type_re.search(res.text) if m is None: return stream_type = m.group("stream_type") if stream_type == "dwlivestream": return self._get_live_streams(res) return self._get_vod_streams(stream_type, res)
validate.optional("play_url"): validate.url(scheme="http"), validate.optional("m3u8_url"): validate.url( scheme="http", path=validate.endswith(".m3u8") ), }, None) }, validate.optional("playerUri"): validate.text, validate.optional("viewerPlusSwfUrl"): validate.url(scheme="http"), validate.optional("lsPlayerSwfUrl"): validate.text, validate.optional("hdPlayerSwfUrl"): validate.text }) _smil_schema = validate.Schema(validate.union({ "http_base": validate.all( validate.xml_find("{http://www.w3.org/2001/SMIL20/Language}head/" "{http://www.w3.org/2001/SMIL20/Language}meta" "[@name='httpBase']"), validate.xml_element(attrib={ "content": validate.text }), validate.get("content") ), "videos": validate.all( validate.xml_findall("{http://www.w3.org/2001/SMIL20/Language}body/" "{http://www.w3.org/2001/SMIL20/Language}switch/" "{http://www.w3.org/2001/SMIL20/Language}video"), [ validate.all( validate.xml_element(attrib={ "src": validate.text, "system-bitrate": validate.all(
class Beam(Plugin): api_url = "https://beam.pro/api/v1/{type}/{id}" channel_manifest = "https://beam.pro/api/v1/channels/{id}/manifest.{type}" _vod_schema = validate.Schema({ "state": "AVAILABLE", "vods": [{ "baseUrl": validate.url(), "data": validate.any(None, { "Height": int }), "format": validate.text }] }, validate.get("vods"), validate.filter(lambda x: x["format"] in ("raw", "hls")), [validate.union({ "url": validate.get("baseUrl"), "format": validate.get("format"), "height": validate.all(validate.get("data"), validate.get("Height")) })]) _assets_schema = validate.Schema( validate.union({ "base": validate.all( validate.xml_find("./head/meta"), validate.get("base"), validate.url(scheme="rtmp") ), "videos": validate.all( validate.xml_findall(".//video"), [ validate.union({ "src": validate.all( validate.get("src"), validate.text ), "height": validate.all( validate.get("height"), validate.text, validate.transform(int) ) }) ] ) }) ) @classmethod def can_handle_url(cls, url): return _url_re.match(url) def _get_vod_stream(self, vod_id): res = http.get(self.api_url.format(type="recordings", id=vod_id)) for sdata in http.json(res, schema=self._vod_schema): if sdata["format"] == "hls": hls_url = urljoin(sdata["url"], "manifest.m3u8") yield "{0}p".format(sdata["height"]), HLSStream(self.session, hls_url) elif sdata["format"] == "raw": raw_url = urljoin(sdata["url"], "source.mp4") yield "{0}p".format(sdata["height"]), HTTPStream(self.session, raw_url) def _get_live_stream(self, channel): res = http.get(self.api_url.format(type="channels", id=channel)) channel_info = http.json(res) if not channel_info["online"]: return res = http.get(self.channel_manifest.format(id=channel_info["id"], type="smil")) assets = http.xml(res, schema=self._assets_schema) for video in assets["videos"]: name = "{0}p".format(video["height"]) stream = RTMPStream(self.session, { "rtmp": "{0}/{1}".format(assets["base"], video["src"]) }) yield name, stream for s in HLSStream.parse_variant_playlist(self.session, self.channel_manifest.format(id=channel_info["id"], type="m3u8")).items(): yield s def _get_streams(self): params = dict(parse_qsl(urlparse(self.url).query)) vod_id = params.get("vod") match = _url_re.match(self.url) channel = match.group("channel") if vod_id: self.logger.debug("Looking for VOD {0} from channel: {1}", vod_id, channel) return self._get_vod_stream(vod_id) else: self.logger.debug("Looking for channel: {0}", channel) return self._get_live_stream(channel)
import re from streamlink.plugin import Plugin from streamlink.plugin.api import http, validate from streamlink.stream import RTMPStream _url_re = re.compile(r"http(s)?://(\w+.)?beam.pro/(?P<channel>[^/]+)") CHANNEL_INFO = "https://beam.pro/api/v1/channels/{0}" CHANNEL_MANIFEST = "https://beam.pro/api/v1/channels/{0}/manifest.smil" _assets_schema = validate.Schema( validate.union({ "base": validate.all( validate.xml_find("./head/meta"), validate.get("base"), validate.url(scheme="rtmp") ), "videos": validate.all( validate.xml_findall(".//video"), [ validate.union({ "src": validate.all( validate.get("src"), validate.text ), "height": validate.all( validate.get("height"), validate.text, validate.transform(int) )
_media_schema = validate.Schema({ "_mediaArray": [{ "_mediaStreamArray": [{ validate.optional("_server"): validate.text, "_stream": validate.any(validate.text, [validate.text]), "_quality": validate.any(int, validate.text) }] }] }) _smil_schema = validate.Schema( validate.union({ "base": validate.all(validate.xml_find("head/meta"), validate.get("base"), validate.url(scheme="http")), "cdn": validate.all(validate.xml_find("head/meta"), validate.get("cdn")), "videos": validate.all(validate.xml_findall("body/seq/video"), [validate.get("src")]) })) class ard_mediathek(Plugin): @classmethod def can_handle_url(cls, url): return _url_re.match(url) def _get_http_streams(self, info):