def test_xml_find(self):
el = Element("parent")
el.append(Element("foo"))
el.append(Element("bar"))
assert validate(xml_find("bar"), el).tag == "bar"
with self.assertRaises(ValueError) as cm:
validate(xml_find("baz"), el)
assert str(cm.exception) == "XPath 'baz' did not return an element"
def _streams_brightcove(self, root):
schema_brightcove = validate.Schema(
validate.any(
validate.all(validate.xml_find(".//*[@accountid][@videoid]"),
validate.union_get("accountid", "videoid")),
validate.all(
validate.xml_find(".//*[@data-account][@data-video-id]"),
validate.union_get("data-account", "data-video-id"))))
try:
account_id, video_id = schema_brightcove.validate(root)
except PluginError:
return
return self._brightcove(account_id, video_id)
def get_wss_api_url(self):
try:
data = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(
".//script[@id='embedded-data'][@data-props]"),
validate.get("data-props"), validate.parse_json(), {
"site": {
"relive": {
"webSocketUrl": validate.url(scheme="wss")
},
validate.optional("frontendId"): int
}
}, validate.get("site"),
validate.union_get(("relive", "webSocketUrl"),
"frontendId")))
except PluginError:
return
wss_api_url, frontend_id = data
if frontend_id is not None:
wss_api_url = update_qsd(wss_api_url, {"frontend_id": frontend_id})
return wss_api_url
def test_failure_schema(self):
with pytest.raises(validate.ValidationError) as cm:
validate.validate(validate.xml_find("."), "not-an-element")
assert_validationerror(
cm.value, """
ValidationError(Callable):
iselement('not-an-element') is not true
""")
def test_failure_not_found(self):
with pytest.raises(validate.ValidationError) as cm:
validate.validate(validate.xml_find("invalid"), Element("foo"))
assert_validationerror(
cm.value, """
ValidationError(xml_find):
XPath 'invalid' did not return an element
""")
def _get_streams(self):
try:
data_url = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//*[@data-ctrl-player]"),
validate.get("data-ctrl-player"),
validate.transform(lambda s: s.replace("'", "\"")),
validate.parse_json(), {"url": validate.text},
validate.get("url")))
except PluginError:
return
data_url = urljoin(self._URL_DATA_BASE, data_url)
log.debug("Player URL: '{0}'", data_url)
self.title, media = self.session.http.get(
data_url,
schema=validate.Schema(
validate.parse_json(name="MEDIAINFO"), {
"mc": {
validate.optional("_title"):
validate.text,
"_mediaArray": [
validate.all(
{
"_mediaStreamArray": [
validate.all(
{
"_quality":
validate.any(
validate.text, int),
"_stream": [validate.url()],
},
validate.union_get(
"_quality", ("_stream", 0)))
]
}, validate.get("_mediaStreamArray"),
validate.transform(dict))
]
}
}, validate.get("mc"),
validate.union_get("_title", ("_mediaArray", 0))))
if media.get("auto"):
for s in HLSStream.parse_variant_playlist(
self.session, media.get("auto")).items():
yield s
else:
for quality, stream in media.items():
yield self._QUALITY_MAP.get(quality, quality), HTTPStream(
self.session, stream)
def _schema_consent(data):
schema_consent = validate.Schema(
validate.parse_html(),
validate.any(
validate.xml_find(".//form[@action='https://consent.youtube.com/s']"),
validate.all(
validate.xml_xpath(".//form[@action='https://consent.youtube.com/save']"),
validate.filter(lambda elem: elem.xpath(".//input[@type='hidden'][@name='set_ytc'][@value='true']")),
validate.get(0),
)
),
validate.union((
validate.get("action"),
validate.xml_xpath(".//input[@type='hidden']"),
)),
)
return schema_consent.validate(data)
def _get_streams(self):
try:
data = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(
".//video[@id='brightcove_video_player']"),
validate.union_get("data-video-id", "data-account",
"data-ad-config-id", "data-player")))
except PluginError:
return
data_video_id, data_account, data_ad_config_id, data_player = data
url = self._PLAYER_URL.format(data_account=data_account,
data_player=data_player)
policy_key = self.session.http.get(
url,
schema=validate.Schema(
validate.transform(self._policy_key_re.search),
validate.any(None, validate.get(1))))
if not policy_key:
return
url = self._API_URL.format(data_account=data_account,
data_video_id=data_video_id)
if data_ad_config_id is not None:
url = update_qsd(url, dict(ad_config_id=data_ad_config_id))
streams = self.session.http.get(
url,
headers={"Accept": f"application/json;pk={policy_key}"},
schema=validate.Schema(
validate.parse_json(), {
"sources": [{
validate.optional("type"): str,
"src": validate.url(),
}],
}, validate.get("sources"),
validate.filter(lambda source: source.get("type") ==
"application/x-mpegURL")))
for stream in streams:
return HLSStream.parse_variant_playlist(self.session,
stream["src"])
def _get_streams(self):
stream_id, has_token, hls_url, dash_url = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//*[@data-video-id]"),
validate.union((
validate.get("data-video-id"),
validate.all(
validate.get("data-video-has-token"),
validate.transform(lambda val: val and val != "false"),
),
validate.get("data-vjs-clip-hls-url"),
validate.get("data-vjs-clip-dash-url"),
)),
),
)
if dash_url and has_token:
token = self._get_stream_token(stream_id, "dash")
parsed = urlsplit(dash_url)
dash_url = urlunsplit(
parsed._replace(path=f"{token}{parsed.path}"))
return DASHStream.parse_manifest(
self.session,
dash_url,
headers={"Referer": self.url},
)
if not hls_url:
return
if has_token:
token = self._get_stream_token(stream_id, "hls")
hls_url = f"{hls_url}?{token}"
return HLSStream.parse_variant_playlist(
self.session,
hls_url,
headers={"Referer": self.url},
)
def test_xml_find(self):
el = Element("parent")
el.append(Element("foo"))
el.append(Element("bar"))
assert validate(xml_find("bar"), el).tag == "bar"
"bitrate": int,
"height": int
}],
validate.optional("play_url"): validate.url(scheme="http"),
validate.optional("m3u8_url"): validate.url(
scheme="http",
path=validate.endswith(".m3u8")
),
}, None)
},
validate.optional("playerUri"): validate.text
})
_smil_schema = validate.Schema(validate.union({
"http_base": validate.all(
validate.xml_find("{http://www.w3.org/2001/SMIL20/Language}head/"
"{http://www.w3.org/2001/SMIL20/Language}meta"
"[@name='httpBase']"),
validate.xml_element(attrib={
"content": validate.text
}),
validate.get("content")
),
"videos": validate.all(
validate.xml_findall("{http://www.w3.org/2001/SMIL20/Language}body/"
"{http://www.w3.org/2001/SMIL20/Language}switch/"
"{http://www.w3.org/2001/SMIL20/Language}video"),
[
validate.all(
validate.xml_element(attrib={
"src": validate.text,
"system-bitrate": validate.all(
def test_xml_find(self):
el = Element("parent")
el.append(Element("foo"))
el.append(Element("bar"))
assert validate(xml_find("bar"), el).tag == "bar"
def _get_streams(self):
schema_metadata = validate.Schema(
validate.parse_json(), {
validate.optional("author"):
validate.all({"name": str}, validate.get("name")),
validate.optional("movie"):
validate.all({"title": str}, validate.get("title")),
validate.optional("hlsManifestUrl"):
validate.url(),
validate.optional("hlsMasterPlaylistUrl"):
validate.url(),
validate.optional("liveDashManifestUrl"):
validate.url(),
validate.optional("videos"): [
validate.all({
"name": str,
"url": validate.url()
}, validate.union_get("name", "url"))
]
})
try:
metadata, metadata_url = self.session.http.get(
self.url,
schema=validate.Schema(
validate.parse_html(),
validate.xml_find(".//*[@data-options]"),
validate.get("data-options"), validate.parse_json(), {
"flashvars": {
validate.optional("metadata"):
str,
validate.optional("metadataUrl"):
validate.all(validate.transform(unquote),
validate.url())
}
}, validate.get("flashvars"),
validate.union_get("metadata", "metadataUrl")))
except PluginError:
log.error("Could not find metadata")
return
self.session.http.headers.update({"Referer": self.url})
if not metadata and metadata_url:
metadata = self.session.http.post(metadata_url).text
log.trace(f"{metadata!r}")
try:
data = schema_metadata.validate(metadata)
except PluginError:
log.error("Could not parse metadata")
return
self.author = data.get("author")
self.title = data.get("movie")
for hls_url in data.get("hlsManifestUrl"), data.get(
"hlsMasterPlaylistUrl"):
if hls_url is not None:
return HLSStream.parse_variant_playlist(self.session, hls_url)
if data.get("liveDashManifestUrl"):
return DASHStream.parse_manifest(self.session,
data.get("liveDashManifestUrl"))
return {
f"{self.QUALITY_WEIGHTS[name]}p" if name in self.QUALITY_WEIGHTS
else name: HTTPStream(self.session, url)
for name, url in data.get("videos", [])
}
_url_re = re.compile(r"http(s)?://(?:(\w+\.)?ardmediathek.de/tv|mediathek.daserste.de/)")
_media_id_re = re.compile(r"/play/(?:media|config)/(\d+)")
_media_schema = validate.Schema({
"_mediaArray": [{
"_mediaStreamArray": [{
validate.optional("_server"): validate.text,
"_stream": validate.any(validate.text, [validate.text]),
"_quality": validate.any(int, validate.text)
}]
}]
})
_smil_schema = validate.Schema(
validate.union({
"base": validate.all(
validate.xml_find("head/meta"),
validate.get("base"),
validate.url(scheme="http")
),
"cdn": validate.all(
validate.xml_find("head/meta"),
validate.get("cdn")
),
"videos": validate.all(
validate.xml_findall("body/seq/video"),
[validate.get("src")]
)
})
)
import re
from streamlink.plugin import Plugin
from streamlink.plugin.api import http, validate
from streamlink.stream import RTMPStream
_url_re = re.compile("http(s)?://(\w+.)?beam.pro/(?P<channel>[^/]+)")
CHANNEL_INFO = "https://beam.pro/api/v1/channels/{0}"
CHANNEL_MANIFEST = "https://beam.pro/api/v1/channels/{0}/manifest.smil"
_assets_schema = validate.Schema(
validate.union({
"base": validate.all(
validate.xml_find("./head/meta"),
validate.get("base"),
validate.url(scheme="rtmp")
),
"videos": validate.all(
validate.xml_findall(".//video"),
[
validate.union({
"src": validate.all(
validate.get("src"),
validate.text
),
"height": validate.all(
validate.get("height"),
validate.text,
validate.transform(int)
)
def test_success(self):
element = Element("foo")
assert validate.validate(validate.xml_find("."), element) is element
_url_re = re.compile(
r"""
http(s)?://(\w+\.)?cybergame.tv
(?:
/videos/(?P<video_id>\d+)
)?
(?:
/(?P<channel>[^/&?]+)
)?
""", re.VERBOSE)
_playlist_schema = validate.Schema(
validate.union({
"base":
validate.all(validate.xml_find("./head/meta"), validate.get("base"),
validate.url(scheme="rtmp")),
"videos":
validate.all(validate.xml_findall(".//video"), [
validate.union({
"src":
validate.all(validate.get("src"), validate.text),
"height":
validate.all(validate.get("height"), validate.text,
validate.transform(int))
})
])
}))
class Cybergame(Plugin):
class DeutscheWelle(Plugin):
default_channel = "1"
url_re = re.compile(r"https?://(?:www\.)?dw\.com/")
channel_re = re.compile(r'''<a.*?data-id="(\d+)".*?class="ici"''')
live_stream_div = re.compile(
r'''
<div\s+class="mediaItem"\s+data-channel-id="(\d+)".*?>.*?
<input\s+type="hidden"\s+name="file_name"\s+value="(.*?)"\s*>.*?<div
''', re.DOTALL | re.VERBOSE)
smil_api_url = "http://www.dw.com/smil/{}"
html5_api_url = "http://www.dw.com/html5Resource/{}"
vod_player_type_re = re.compile(
r'<input type="hidden" name="player_type" value="(?P<stream_type>.+?)">'
)
stream_vod_data_re = re.compile(
r'<input\s+type="hidden"\s+name="file_name"\s+value="(?P<stream_url>.+?)">.*?'
r'<input\s+type="hidden"\s+name="media_id"\s+value="(?P<stream_id>\d+)">',
re.DOTALL)
smil_schema = validate.Schema(
validate.union({
"base":
validate.all(validate.xml_find(".//meta"),
validate.xml_element(attrib={"base": validate.text}),
validate.get("base")),
"streams":
validate.all(validate.xml_findall(".//switch/*"), [
validate.all(
validate.getattr("attrib"), {
"src":
validate.text,
"system-bitrate":
validate.all(
validate.text,
validate.transform(int),
),
validate.optional("width"):
validate.all(validate.text, validate.transform(int))
})
])
}))
@classmethod
def can_handle_url(cls, url):
return cls.url_re.match(url) is not None
def _create_stream(self, url, quality=None):
if url.startswith('rtmp://'):
return (quality, RTMPStream(self.session, {'rtmp': url}))
if url.endswith('.m3u8'):
return HLSStream.parse_variant_playlist(self.session, url).items()
return (quality, HTTPStream(self.session, url))
def _get_live_streams(self, page):
# check if a different language has been selected
qs = dict(parse_qsl(urlparse(self.url).query))
channel = qs.get("channel")
if not channel:
m = self.channel_re.search(page.text)
channel = m and m.group(1)
self.logger.debug("Using sub-channel ID: {0}", channel)
# extract the streams from the page, mapping between channel-id and stream url
media_items = self.live_stream_div.finditer(page.text)
stream_map = dict([mi.groups((1, 2)) for mi in media_items])
stream_url = stream_map.get(str(channel) or self.default_channel)
if stream_url:
return self._create_stream(stream_url)
def _get_vod_streams(self, stream_type, page):
m = self.stream_vod_data_re.search(page.text)
if m is None:
return
stream_url, stream_id = m.groups()
if stream_type == "video":
stream_api_id = "v-{}".format(stream_id)
default_quality = "vod"
elif stream_type == "audio":
stream_api_id = "a-{}".format(stream_id)
default_quality = "audio"
else:
return
# Retrieve stream embedded in web page
yield self._create_stream(stream_url, default_quality)
# Retrieve streams using API
res = self.session.http.get(self.smil_api_url.format(stream_api_id))
videos = self.session.http.xml(res, schema=self.smil_schema)
for video in videos['streams']:
url = videos["base"] + video["src"]
if url == stream_url or url.replace("_dwdownload.",
".") == stream_url:
continue
if video["system-bitrate"] > 0:
# If width is available, use it to select the best stream
# amongst those with same bitrate
quality = "{}k".format(
(video["system-bitrate"] + video.get("width", 0)) // 1000)
else:
quality = default_quality
yield self._create_stream(url, quality)
def _get_streams(self):
res = self.session.http.get(self.url)
m = self.vod_player_type_re.search(res.text)
if m is None:
return
stream_type = m.group("stream_type")
if stream_type == "dwlivestream":
return self._get_live_streams(res)
return self._get_vod_streams(stream_type, res)
validate.optional("play_url"): validate.url(scheme="http"),
validate.optional("m3u8_url"): validate.url(
scheme="http",
path=validate.endswith(".m3u8")
),
}, None)
},
validate.optional("playerUri"): validate.text,
validate.optional("viewerPlusSwfUrl"): validate.url(scheme="http"),
validate.optional("lsPlayerSwfUrl"): validate.text,
validate.optional("hdPlayerSwfUrl"): validate.text
})
_smil_schema = validate.Schema(validate.union({
"http_base": validate.all(
validate.xml_find("{http://www.w3.org/2001/SMIL20/Language}head/"
"{http://www.w3.org/2001/SMIL20/Language}meta"
"[@name='httpBase']"),
validate.xml_element(attrib={
"content": validate.text
}),
validate.get("content")
),
"videos": validate.all(
validate.xml_findall("{http://www.w3.org/2001/SMIL20/Language}body/"
"{http://www.w3.org/2001/SMIL20/Language}switch/"
"{http://www.w3.org/2001/SMIL20/Language}video"),
[
validate.all(
validate.xml_element(attrib={
"src": validate.text,
"system-bitrate": validate.all(
class Beam(Plugin):
api_url = "https://beam.pro/api/v1/{type}/{id}"
channel_manifest = "https://beam.pro/api/v1/channels/{id}/manifest.{type}"
_vod_schema = validate.Schema({
"state": "AVAILABLE",
"vods": [{
"baseUrl": validate.url(),
"data": validate.any(None, {
"Height": int
}),
"format": validate.text
}]
},
validate.get("vods"),
validate.filter(lambda x: x["format"] in ("raw", "hls")),
[validate.union({
"url": validate.get("baseUrl"),
"format": validate.get("format"),
"height": validate.all(validate.get("data"), validate.get("Height"))
})])
_assets_schema = validate.Schema(
validate.union({
"base": validate.all(
validate.xml_find("./head/meta"),
validate.get("base"),
validate.url(scheme="rtmp")
),
"videos": validate.all(
validate.xml_findall(".//video"),
[
validate.union({
"src": validate.all(
validate.get("src"),
validate.text
),
"height": validate.all(
validate.get("height"),
validate.text,
validate.transform(int)
)
})
]
)
})
)
@classmethod
def can_handle_url(cls, url):
return _url_re.match(url)
def _get_vod_stream(self, vod_id):
res = http.get(self.api_url.format(type="recordings", id=vod_id))
for sdata in http.json(res, schema=self._vod_schema):
if sdata["format"] == "hls":
hls_url = urljoin(sdata["url"], "manifest.m3u8")
yield "{0}p".format(sdata["height"]), HLSStream(self.session, hls_url)
elif sdata["format"] == "raw":
raw_url = urljoin(sdata["url"], "source.mp4")
yield "{0}p".format(sdata["height"]), HTTPStream(self.session, raw_url)
def _get_live_stream(self, channel):
res = http.get(self.api_url.format(type="channels", id=channel))
channel_info = http.json(res)
if not channel_info["online"]:
return
res = http.get(self.channel_manifest.format(id=channel_info["id"], type="smil"))
assets = http.xml(res, schema=self._assets_schema)
for video in assets["videos"]:
name = "{0}p".format(video["height"])
stream = RTMPStream(self.session, {
"rtmp": "{0}/{1}".format(assets["base"], video["src"])
})
yield name, stream
for s in HLSStream.parse_variant_playlist(self.session,
self.channel_manifest.format(id=channel_info["id"], type="m3u8")).items():
yield s
def _get_streams(self):
params = dict(parse_qsl(urlparse(self.url).query))
vod_id = params.get("vod")
match = _url_re.match(self.url)
channel = match.group("channel")
if vod_id:
self.logger.debug("Looking for VOD {0} from channel: {1}", vod_id, channel)
return self._get_vod_stream(vod_id)
else:
self.logger.debug("Looking for channel: {0}", channel)
return self._get_live_stream(channel)
import re
from streamlink.plugin import Plugin
from streamlink.plugin.api import http, validate
from streamlink.stream import RTMPStream
_url_re = re.compile(r"http(s)?://(\w+.)?beam.pro/(?P<channel>[^/]+)")
CHANNEL_INFO = "https://beam.pro/api/v1/channels/{0}"
CHANNEL_MANIFEST = "https://beam.pro/api/v1/channels/{0}/manifest.smil"
_assets_schema = validate.Schema(
validate.union({
"base": validate.all(
validate.xml_find("./head/meta"),
validate.get("base"),
validate.url(scheme="rtmp")
),
"videos": validate.all(
validate.xml_findall(".//video"),
[
validate.union({
"src": validate.all(
validate.get("src"),
validate.text
),
"height": validate.all(
validate.get("height"),
validate.text,
validate.transform(int)
)
_media_schema = validate.Schema({
"_mediaArray": [{
"_mediaStreamArray": [{
validate.optional("_server"):
validate.text,
"_stream":
validate.any(validate.text, [validate.text]),
"_quality":
validate.any(int, validate.text)
}]
}]
})
_smil_schema = validate.Schema(
validate.union({
"base":
validate.all(validate.xml_find("head/meta"), validate.get("base"),
validate.url(scheme="http")),
"cdn":
validate.all(validate.xml_find("head/meta"), validate.get("cdn")),
"videos":
validate.all(validate.xml_findall("body/seq/video"),
[validate.get("src")])
}))
class ard_mediathek(Plugin):
@classmethod
def can_handle_url(cls, url):
return _url_re.match(url)
def _get_http_streams(self, info):
