def test_dict_optional_keys(self):
assert validate({"a": 1, optional("b"): 2}, {"a": 1}) == {"a": 1}
assert validate({
"a": 1,
optional("b"): 2
}, {
"a": 1,
"b": 2
}) == {
"a": 1,
"b": 2
}
class Euronews(Plugin):
_url_re = re.compile(r"http(?:s)?://(\w+)\.?euronews.com/(live|.*)")
_re_vod = re.compile(
r'<meta\s+property="og:video"\s+content="(http.*?)"\s*/>')
_live_api_url = "http://{0}.euronews.com/api/watchlive.json"
_live_schema = validate.Schema({u"url": validate.url()})
_stream_api_schema = validate.Schema({
u'status':
u'ok',
u'primary':
validate.url(),
validate.optional(u'backup'):
validate.url()
})
@classmethod
def can_handle_url(cls, url):
return cls._url_re.match(url)
def _get_vod_stream(self):
"""
Find the VOD video url
:return: video url
"""
res = http.get(self.url)
video_urls = self._re_vod.findall(res.text)
if len(video_urls):
return dict(vod=HTTPStream(self.session, video_urls[0]))
def _get_live_streams(self, subdomain):
"""
Get the live stream in a particular language
:param subdomain:
:return:
"""
res = http.get(self._live_api_url.format(subdomain))
live_res = http.json(res, schema=self._live_schema)
api_res = http.get(live_res[u"url"])
stream_data = http.json(api_res, schema=self._stream_api_schema)
return HLSStream.parse_variant_playlist(self.session,
stream_data[u'primary'])
def _get_streams(self):
"""
Find the streams for euronews
:return:
"""
match = self._url_re.match(self.url)
subdomain, path = match.groups()
if path == "live":
return self._get_live_streams(subdomain)
else:
return self._get_vod_stream()
class Aftonbladet(Plugin):
"""Plugin for swedish news paper Aftonbladet's streaming service."""
_url_re = re.compile(
r"https?://tv\.aftonbladet\.se/[^/]+/[^/]+/(?P<id>\d+)")
api_url = "https://svp.vg.no/svp/api/v1/ab/assets/{0}?appName=svp-player"
_video_schema = validate.Schema({
validate.optional("title"): validate.text,
validate.optional("assetType"): validate.text,
validate.optional("streamType"): validate.text,
validate.optional("status"): validate.text,
"streamUrls": {
validate.optional("hls"): validate.any(validate.text, None),
validate.optional("hds"): validate.any(validate.text, None),
validate.optional("mp4"): validate.any(validate.text, None),
},
})
@classmethod
def can_handle_url(cls, url):
return cls._url_re.match(url)
def _get_streams(self):
m_url = self._url_re.match(self.url)
if not m_url:
return
video_id = m_url.group("id")
headers = {"User-Agent": useragents.FIREFOX, "Referer": self.url}
res = http.get(self.api_url.format(video_id), headers=headers)
data = http.json(res, schema=self._video_schema)
title = data.get("title")
streamurls = data.get("streamUrls")
if title:
self.stream_title = title
if streamurls:
hls_url = streamurls.get("hls")
if hls_url:
streams = HLSStream.parse_variant_playlist(
self.session, hls_url, headers=headers).items()
if not streams:
yield "live", HLSStream(self.session,
hls_url,
headers=headers)
for s in streams:
yield s
hds_url = streamurls.get("hds")
if hds_url:
for s in HDSStream.parse_manifest(self.session,
hds_url,
headers=headers).items():
yield s
mp4_url = streamurls.get("mp4")
if mp4_url:
name = "live"
yield name, HTTPStream(self.session, mp4_url, headers=headers)
BALANCER_URL = "http://www.mips.tv:1935/loadbalancer"
PLAYER_URL = "http://mips.tv/embedplayer/{0}/1/500/400"
SWF_URL = "http://mips.tv/content/scripts/eplayer.swf"
_url_re = re.compile(r"http(s)?://(\w+.)?mips.tv/(?P<channel>[^/&?]+)")
_flashvars_re = re.compile(r"'FlashVars', '([^']+)'")
_rtmp_re = re.compile(r"redirect=(.+)")
_schema = validate.Schema(
validate.transform(_flashvars_re.search),
validate.any(
None,
validate.all(validate.get(1), validate.transform(parse_query), {
"id": validate.transform(int),
validate.optional("s"): validate.text
})))
_rtmp_schema = validate.Schema(
validate.transform(_rtmp_re.search),
validate.get(1),
)
class Mips(Plugin):
@classmethod
def can_handle_url(self, url):
return _url_re.match(url)
@Plugin.broken()
def _get_streams(self):
match = _url_re.match(self.url)
_access_token_schema = validate.Schema(
{
"token": validate.text,
"sig": validate.text
}, validate.union((validate.get("sig"), validate.get("token"))))
_token_schema = validate.Schema(
{
"chansub": {
"restricted_bitrates":
validate.all([validate.text],
validate.filter(lambda n: not re.match(
r"(.+_)?archives|live|chunked", n)))
}
}, validate.get("chansub"))
_user_schema = validate.Schema(
{validate.optional("display_name"): validate.text},
validate.get("display_name"))
_video_schema = validate.Schema({
"chunks": {
validate.text: [{
"length": int,
"url": validate.any(None, validate.url(scheme="http")),
"upkeep": validate.any("pass", "fail", None)
}]
},
"restrictions": {
validate.text: validate.text
},
"start_offset": int,
"end_offset": int,
})
def parse_fmt_list(formatsmap):
formats = {}
if not formatsmap:
return formats
for format in formatsmap.split(","):
s = format.split("/")
(w, h) = s[1].split("x")
formats[int(s[0])] = "{0}p".format(h)
return formats
_config_schema = validate.Schema(
{
validate.optional("fmt_list"): validate.all(
validate.text,
validate.transform(parse_fmt_list)
),
validate.optional("url_encoded_fmt_stream_map"): validate.all(
validate.text,
validate.transform(parse_stream_map),
[{
"itag": validate.all(
validate.text,
validate.transform(int)
),
"quality": validate.text,
"url": validate.url(scheme="http"),
validate.optional("s"): validate.text,
validate.optional("stereo3d"): validate.all(
class Dogan(Plugin):
"""
Support for the live streams from Doğan Media Group channels
"""
url_re = re.compile(
r"""
https?://(?:www.)?
(?:teve2.com.tr/(?:canli-yayin|filmler/.*|programlar/.*)|
kanald.com.tr/.*|
cnnturk.com/canli-yayin|
dreamtv.com.tr/canli-yayin|
dreamturk.com.tr/canli)
""", re.VERBOSE)
playerctrl_re = re.compile(
r'''<div[^>]*?ng-controller=(?P<quote>["'])(?:Live)?PlayerCtrl(?P=quote).*?>''',
re.DOTALL)
videoelement_re = re.compile(
r'''<div[^>]*?id=(?P<quote>["'])video-element(?P=quote).*?>''',
re.DOTALL)
data_id_re = re.compile(
r'''data-id=(?P<quote>["'])(?P<id>\w+)(?P=quote)''')
content_id_re = re.compile(r'"content(?:I|i)d", "(\w+)"')
content_api = "/actions/content/media/{id}"
new_content_api = "/action/media/{id}"
content_api_schema = validate.Schema({
"Id": validate.text,
"Media": {
"Link": {
"DefaultServiceUrl": validate.url(),
validate.optional("ServiceUrl"):
validate.any(validate.url(), ""),
"SecurePath": validate.text,
}
}
})
@classmethod
def can_handle_url(cls, url):
return cls.url_re.match(url) is not None
def _get_content_id(self):
res = http.get(self.url)
# find the contentId
content_id_m = self.content_id_re.search(res.text)
if content_id_m:
return content_id_m.group(1)
# find the PlayerCtrl div
player_ctrl_m = self.playerctrl_re.search(res.text)
if player_ctrl_m:
# extract the content id from the player control data
player_ctrl_div = player_ctrl_m.group(0)
content_id_m = self.data_id_re.search(player_ctrl_div)
if content_id_m:
return content_id_m.group("id")
# find <div id="video-element"
videoelement_m = self.videoelement_re.search(res.text)
if videoelement_m:
# extract the content id from the player control data
videoelement_div = videoelement_m.group(0)
content_id_m = self.data_id_re.search(videoelement_div)
if content_id_m:
return content_id_m.group("id")
def _get_hls_url(self, content_id):
# make the api url relative to the current domain
if "cnnturk" in self.url or "teve2.com.tr" in self.url:
self.logger.debug("Using new content API url")
api_url = urljoin(self.url,
self.new_content_api.format(id=content_id))
else:
api_url = urljoin(self.url, self.content_api.format(id=content_id))
apires = http.get(api_url)
stream_data = http.json(apires, schema=self.content_api_schema)
d = stream_data["Media"]["Link"]
return urljoin((d["ServiceUrl"] or d["DefaultServiceUrl"]),
d["SecurePath"])
def _get_streams(self):
content_id = self._get_content_id()
if content_id:
self.logger.debug(u"Loading content: {}", content_id)
hls_url = self._get_hls_url(content_id)
return HLSStream.parse_variant_playlist(self.session, hls_url)
else:
self.logger.error(u"Could not find the contentId for this stream")
class VGTV(Plugin):
"""
Plugin for VGTV, Norwegian newspaper VG Nett's streaming service.
Plugin for swedish news paper Aftonbladet's streaming service.
"""
_url_re = re.compile(r"""https?://(?:www\.)?
(?P<host>
tv\.aftonbladet\.se/abtv
|
(?:ap\.)?vgtv\.no
)
(?:/webtv(?:[^/]+)?)?
/[^/]+/(?P<id>\d+)
""", re.VERBOSE)
appname_map = {
"ap.vgtv.no": "aptv",
"tv.aftonbladet.se/abtv": "abtv",
"vgtv.no": "vgtv",
}
apiname_map = {
"abtv": "ab",
"aptv": "ap",
"vgtv": "vgtv",
}
api_url = "https://svp.vg.no/svp/api/v1/{0}/assets/{1}?appName={2}-website"
_video_schema = validate.Schema(
{
validate.optional("title"): validate.text,
validate.optional("assetType"): validate.text,
validate.optional("streamType"): validate.text,
validate.optional("status"): validate.text,
"streamUrls": {
validate.optional("hls"): validate.any(validate.text, None),
validate.optional("hds"): validate.any(validate.text, None),
validate.optional("mp4"): validate.any(validate.text, None),
},
}
)
@classmethod
def can_handle_url(cls, url):
return cls._url_re.match(url)
def _get_streams(self):
m_url = self._url_re.match(self.url)
if not m_url:
return
video_id = m_url.group("id")
appname = self.appname_map[m_url.group("host")]
apiname = self.apiname_map[appname]
headers = {
"User-Agent": useragents.FIREFOX,
"Referer": self.url
}
res = http.get(self.api_url.format(apiname, video_id, appname), headers=headers)
data = http.json(res, schema=self._video_schema)
title = data.get("title")
streamurls = data.get("streamUrls")
if title:
self.stream_title = title
if streamurls:
hls_url = streamurls.get("hls")
if hls_url:
self.logger.debug("HLS URL: {0}".format(hls_url))
streams = HLSStream.parse_variant_playlist(self.session, hls_url, headers=headers).items()
if not streams:
yield "live", HLSStream(self.session, hls_url, headers=headers)
for s in streams:
yield s
hds_url = streamurls.get("hds")
if hds_url:
self.logger.debug("HDS URL: {0}".format(hds_url))
for s in HDSStream.parse_manifest(self.session, hds_url, headers=headers).items():
yield s
mp4_url = streamurls.get("mp4")
if mp4_url:
self.logger.debug("MP4 URL: {0}".format(mp4_url))
name = "live"
yield name, HTTPStream(self.session, mp4_url, headers=headers)
class LiveEdu(Plugin):
login_url = "https://www.liveedu.tv/accounts/login/"
url_re = re.compile(r"https?://(?:\w+\.)?(?:livecoding|liveedu)\.tv/")
config_re = re.compile(
r"""\Wconfig.(?P<key>\w+)\s*=\s*(?P<q>['"])(?P<value>.*?)(?P=q);""")
csrf_re = re.compile(r'''"csrfToken"\s*:\s*"(\w+)"''')
api_schema = validate.Schema({
"viewing_urls": {
validate.optional("error"):
validate.text,
validate.optional("urls"): [{
"src":
validate.url(),
"type":
validate.text,
validate.optional("res"):
int,
validate.optional("label"):
validate.text,
}]
}
})
config_schema = validate.Schema({
"selectedVideoHID": validate.text,
"livestreamURL": validate.text,
"videosURL": validate.text
})
options = PluginOptions({"email": None, "password": None})
@classmethod
def can_handle_url(cls, url):
return cls.url_re.match(url) is not None
def login(self):
"""
Attempt a login to LiveEdu.tv
"""
email = self.get_option("email")
password = self.get_option("password")
if email and password:
res = http.get(self.login_url)
csrf_match = self.csrf_re.search(res.text)
token = csrf_match and csrf_match.group(1)
self.logger.debug("Attempting login as {0} (token={1})", email,
token)
res = http.post(self.login_url,
data=dict(login=email,
password=password,
csrfmiddlewaretoken=token),
allow_redirects=False,
raise_for_status=False,
headers={"Referer": self.login_url})
if res.status_code != 302:
self.logger.error("Failed to login to LiveEdu account: {0}",
email)
def _get_streams(self):
"""
Get the config object from the page source and call the
API to get the list of streams
:return:
"""
# attempt a login
self.login()
res = http.get(self.url)
# decode the config for the page
matches = self.config_re.finditer(res.text)
try:
config = self.config_schema.validate(
dict([m.group("key", "value") for m in matches]))
except PluginError:
return
if config["selectedVideoHID"]:
self.logger.debug("Found video hash ID: {0}",
config["selectedVideoHID"])
api_url = urljoin(
self.url,
urljoin(config["videosURL"], config["selectedVideoHID"]))
elif config["livestreamURL"]:
self.logger.debug("Found live stream URL: {0}",
config["livestreamURL"])
api_url = urljoin(self.url, config["livestreamURL"])
else:
return
ares = http.get(api_url)
data = http.json(ares, schema=self.api_schema)
viewing_urls = data["viewing_urls"]
if "error" in viewing_urls:
self.logger.error("Failed to load streams: {0}",
viewing_urls["error"])
else:
for url in viewing_urls["urls"]:
try:
label = "{0}p".format(url.get("res", url["label"]))
except KeyError:
label = "live"
if url["type"] == "rtmp/mp4" and RTMPStream.is_usable(
self.session):
params = {
"rtmp": url["src"],
"pageUrl": self.url,
"live": True,
}
yield label, RTMPStream(self.session, params)
elif url["type"] == "application/x-mpegURL":
for s in HLSStream.parse_variant_playlist(
self.session, url["src"]).items():
yield s
"vod": True,
"last_update": "2017-05-03",
}
MEDIA_URL = "http://www.ardmediathek.de/play/media/{0}"
SWF_URL = "http://www.ardmediathek.de/ard/static/player/base/flash/PluginFlash.swf"
HDCORE_PARAMETER = "?hdcore=3.3.0"
QUALITY_MAP = {"auto": "auto", 3: "544p", 2: "360p", 1: "288p", 0: "144p"}
_url_re = re.compile(
r"http(s)?://(?:(\w+\.)?ardmediathek.de/tv|mediathek.daserste.de/)")
_media_id_re = re.compile(r"/play/(?:media|config)/(\d+)")
_media_schema = validate.Schema({
"_mediaArray": [{
"_mediaStreamArray": [{
validate.optional("_server"):
validate.text,
"_stream":
validate.any(validate.text, [validate.text]),
"_quality":
validate.any(int, validate.text)
}]
}]
})
_smil_schema = validate.Schema(
validate.union({
"base":
validate.all(validate.xml_find("head/meta"), validate.get("base"),
validate.url(scheme="http")),
"cdn":
validate.all(validate.xml_find("head/meta"), validate.get("cdn")),
class OKru(Plugin):
"""Livecli Plugin for ok.ru"""
_data_re = re.compile(
r"""data-options=(?P<q>["'])(?P<data>{[^"']+})(?P=q)""")
_url_re = re.compile(r"""https?://(?:www\.)?ok\.ru/""")
_metadata_schema = validate.Schema(
validate.transform(parse_json),
validate.any(
{
"videos":
validate.any([], [{
"name": validate.text,
"url": validate.text,
}]),
validate.optional("hlsManifestUrl"):
validate.text,
validate.optional("hlsMasterPlaylistUrl"):
validate.text,
validate.optional("liveDashManifestUrl"):
validate.text,
validate.optional("rtmpUrl"):
validate.text,
}, None))
_data_schema = validate.Schema(
validate.all(
validate.transform(_data_re.search), validate.get("data"),
validate.transform(unescape), validate.transform(parse_json),
validate.get("flashvars"),
validate.any({"metadata": _metadata_schema},
{"metadataUrl": validate.transform(unquote)}, None)))
QUALITY_WEIGHTS = {
"full": 1080,
"1080": 1080,
"hd": 720,
"720": 720,
"sd": 480,
"480": 480,
"360": 360,
"low": 360,
"lowest": 240,
"mobile": 144,
}
@classmethod
def can_handle_url(cls, url):
return cls._url_re.match(url)
@classmethod
def stream_weight(cls, key):
weight = cls.QUALITY_WEIGHTS.get(key)
if weight:
return weight, "okru"
return Plugin.stream_weight(key)
def _get_streams(self):
headers = {"User-Agent": useragents.FIREFOX, "Referer": self.url}
data = http.get(self.url, headers=headers, schema=self._data_schema)
metadata = data.get("metadata")
metadata_url = data.get("metadataUrl")
if metadata_url:
metadata = http.post(metadata_url,
headers=headers,
schema=self._metadata_schema)
if metadata:
list_hls = [
metadata.get("hlsManifestUrl"),
metadata.get("hlsMasterPlaylistUrl"),
]
for hls_url in list_hls:
if hls_url is not None:
for s in HLSStream.parse_variant_playlist(
self.session, hls_url, headers=headers).items():
yield s
if metadata.get("videos"):
for http_stream in metadata.get("videos"):
http_name = http_stream["name"]
http_url = http_stream["url"]
yield http_name, HTTPStream(self.session,
http_url,
headers=headers)
if metadata.get("rtmpUrl"):
yield "live", RTMPStream(
self.session, params={"rtmp": metadata.get("rtmpUrl")})
_room_id_re = re.compile(r'"roomId":(?P<room_id>\d+),')
_room_id_alt_re = re.compile(
r'content="showroom:///room\?room_id=(?P<room_id>\d+)"')
_room_id_lookup_failure_log = 'Failed to find room_id for {0} using {1} regex'
_api_status_url = 'https://www.showroom-live.com/room/is_live?room_id={room_id}'
_api_stream_url = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}'
_api_stream_schema = validate.Schema(
validate.any(
{
"streaming_url_list":
validate.all([{
"url": validate.text,
validate.optional("stream_name"): validate.text,
"id": int,
"label": validate.text,
"is_default": int,
"type": validate.text,
"quality": int,
}])
}, {}))
# the "low latency" streams are rtmp, the others are hls
_rtmp_quality_lookup = {
"オリジナル画質": "high",
"オリジナル画質(低遅延)": "high",
"original spec(low latency)": "high",
"original spec": "high",
"低画質": "low",
http(s)?://(\w+\.)?
dailymotion.com
(?:
(/embed)?/(video|live)
/(?P<media_id>[^_?/]+)
|
/(?P<channel_name>[A-Za-z0-9-_]+)
)
""", re.VERBOSE)
chromecast_re = re.compile(r'''stream_chromecast_url"\s*:\s*(?P<url>".*?")''')
_media_inner_schema = validate.Schema([{
"layerList": [{
"name":
validate.text,
validate.optional("sequenceList"): [{
"layerList":
validate.all([{
"name": validate.text,
validate.optional("param"): dict
}],
validate.filter(lambda l: l["name"] in
("video", "reporting", "message")))
}]
}]
}])
_media_schema = validate.Schema(
validate.any(
_media_inner_schema,
validate.all({"sequence": _media_inner_schema},
validate.get("sequence"))))
class OlympicChannel(Plugin):
_url_re = re.compile(
r"http(?:s)?://(\w+)\.?olympicchannel.com/../(?P<type>tv|playback)/(livestream-.\d|.*)/"
)
_live_api_url = "https://www.olympicchannel.com{0}api/v2/metadata/{1}"
_stream_get_url = "https://www.olympicchannel.com/en/proxy/viewings/"
_stream_api_schema = validate.Schema({
u'status':
u'ok',
u'primary':
validate.url(),
validate.optional(u'backup'):
validate.url()
})
@classmethod
def can_handle_url(cls, url):
return cls._url_re.match(url)
def _get_vod_streams(self):
page = http.get(self.url)
asset = re.search(r'asse_.{32}', str(page._content)).group(0)
post_data = '{"asset_url":"/api/assets/%s/"}' % asset
stream_data = http.json(
http.post(self._stream_get_url,
data=post_data))['objects'][0]['level3']['streaming_url']
return HLSStream.parse_variant_playlist(self.session, stream_data)
def _get_live_streams(self, lang, path):
"""
Get the live stream in a particular language
:param lang:
:param path:
:return:
"""
res = http.get(self._live_api_url.format(lang, path))
live_res = http.json(res)['default']['uid']
post_data = '{"channel_url":"/api/channels/%s/"}' % live_res
try:
stream_data = http.json(
http.post(self._stream_get_url, data=post_data))['stream_url']
except BaseException:
stream_data = http.json(
http.post(self._stream_get_url, data=post_data))['channel_url']
return HLSStream.parse_variant_playlist(self.session, stream_data)
def _get_streams(self):
"""
Find the streams for OlympicChannel
:return:
"""
match = self._url_re.match(self.url)
type_of_stream = match.group('type')
lang = re.search(r"/../", self.url).group(0)
if type_of_stream == 'tv':
path = re.search(r"tv/.*-\d/$", self.url).group(0)
return self._get_live_streams(lang, path)
elif type_of_stream == 'playback':
path = re.search(r"/playback/.*/$", self.url).group(0)
return self._get_vod_streams()
_flashvar_re = re.compile(r"""(['"])(.*?)\1\s*:\s*(['"])(.*?)\3""")
_clientlibs_re = re.compile(r"""<script.*?src=(['"])(.*?/clientlibs_anime_watch.*?\.js)\1""")
_schema = validate.Schema(
validate.union({
"flashvars": validate.all(
validate.transform(_flashvars_re.search),
validate.get(1),
validate.transform(_flashvar_re.findall),
validate.map(lambda v: (v[1], v[3])),
validate.transform(dict),
{
"s": validate.text,
"country": validate.text,
"init": validate.text,
validate.optional("ss_id"): validate.text,
validate.optional("mv_id"): validate.text,
validate.optional("device_cd"): validate.text,
validate.optional("ss1_prm"): validate.text,
validate.optional("ss2_prm"): validate.text,
validate.optional("ss3_prm"): validate.text
}
),
"clientlibs": validate.all(
validate.transform(_clientlibs_re.search),
validate.get(2),
validate.text
)
})
)
class AdultSwim(Plugin):
API_URL = "http://www.adultswim.com/videos/api/v2/videos/{id}?fields=stream"
vod_api = "http://www.adultswim.com/videos/api/v0/assets"
url_re = re.compile(
r"""https?://(?:www\.)?adultswim\.com/videos
(?:/(streams))?
(?:/([^/]+))?
(?:/([^/]+))?
""", re.VERBOSE)
_stream_data_re = re.compile(r"(?:__)?AS_INITIAL_DATA(?:__)? = (\{.*?});",
re.M | re.DOTALL)
live_schema = validate.Schema({
u"streams": {
validate.text: {
u"stream":
validate.text,
validate.optional(u"isLive"):
bool,
u"archiveEpisodes": [{
u"id": validate.text,
u"slug": validate.text,
}]
}
}
})
vod_id_schema = validate.Schema(
{u"show": {
u"sluggedVideo": {
u"id": validate.text
}
}}, validate.transform(lambda x: x["show"]["sluggedVideo"]["id"]))
_api_schema = validate.Schema({
u'status': u'ok',
u'data': {
u'stream': {
u'assets': [{
u'url': validate.url()
}]
}
}
})
_vod_api_schema = validate.Schema(
validate.all(validate.xml_findall(".//files/file"), [
validate.xml_element,
validate.transform(lambda v: {
"bitrate": v.attrib.get("bitrate"),
"url": v.text
})
]))
@classmethod
def can_handle_url(cls, url):
match = AdultSwim.url_re.match(url)
return match is not None
def _make_hls_hds_stream(self, func, stream, *args, **kwargs):
return func(self.session, stream["url"], *args, **kwargs)
def _get_show_streams(self,
stream_data,
show,
episode,
platform="desktop"):
video_id = parse_json(stream_data.group(1), schema=self.vod_id_schema)
res = http.get(self.vod_api,
params={
"platform": platform,
"id": video_id
})
# create a unique list of the stream manifest URLs
streams = []
urldups = []
for stream in parse_xml(res.text, schema=self._vod_api_schema):
if stream["url"] not in urldups:
streams.append(stream)
urldups.append(stream["url"])
mapper = StreamMapper(lambda fmt, strm: strm["url"].endswith(fmt))
mapper.map(".m3u8", self._make_hls_hds_stream,
HLSStream.parse_variant_playlist)
mapper.map(".f4m",
self._make_hls_hds_stream,
HDSStream.parse_manifest,
is_akamai=True)
mapper.map(
".mp4", lambda s:
(s["bitrate"] + "k", HTTPStream(self.session, s["url"])))
for q, s in mapper(streams):
yield q, s
def _get_live_stream(self, stream_data, show, episode=None):
# parse the stream info as json
stream_info = parse_json(stream_data.group(1), schema=self.live_schema)
# get the stream ID
stream_id = None
show_info = stream_info[u"streams"][show]
if episode:
self.logger.debug("Loading replay of episode: {0}/{1}", show,
episode)
for epi in show_info[u"archiveEpisodes"]:
if epi[u"slug"] == episode:
stream_id = epi[u"id"]
elif show_info.get("isLive") or not len(show_info[u"archiveEpisodes"]):
self.logger.debug("Loading LIVE streams for: {0}", show)
stream_id = show_info[u"stream"]
else: # off-air
if len(show_info[u"archiveEpisodes"]):
epi = show_info[u"archiveEpisodes"][0]
self.logger.debug("Loading replay of episode: {0}/{1}", show,
epi[u"slug"])
stream_id = epi[u"id"]
else:
self.logger.error("This stream is currently offline")
return
if stream_id:
api_url = self.API_URL.format(id=stream_id)
res = http.get(api_url,
headers={"User-Agent": useragents.SAFARI_8})
stream_data = http.json(res, schema=self._api_schema)
mapper = StreamMapper(lambda fmt, surl: surl.endswith(fmt))
mapper.map(".m3u8", HLSStream.parse_variant_playlist, self.session)
mapper.map(".f4m", HDSStream.parse_manifest, self.session)
stream_urls = [
asset[u"url"]
for asset in stream_data[u'data'][u'stream'][u'assets']
]
for q, s in mapper(stream_urls):
yield q, s
else:
self.logger.error(
"Couldn't find the stream ID for this stream: {0}".format(
show))
def _get_streams(self):
# get the page
url_match = self.url_re.match(self.url)
live_stream, show_name, episode_name = url_match.groups()
if live_stream:
show_name = show_name or "live-stream"
res = http.get(self.url, headers={"User-Agent": useragents.SAFARI_8})
# find the big blob of stream info in the page
stream_data = self._stream_data_re.search(res.text)
if stream_data:
if live_stream:
streams = self._get_live_stream(stream_data, show_name,
episode_name)
else:
self.logger.debug("Loading VOD streams for: {0}/{1}",
show_name, episode_name)
streams = self._get_show_streams(stream_data, show_name,
episode_name)
# De-dup the streams, some of the mobile streams overlap the desktop streams
dups = set()
for q, s in streams:
if hasattr(s, "args") and "url" in s.args:
if s.args["url"] not in dups:
yield q, s
dups.add(s.args["url"])
else:
yield q, s
else:
self.logger.error(
"Couldn't find the stream data for this stream: {0}".format(
show_name))
validate.any(
None, {
"status":
int,
"media": [{
"duration":
validate.any(float, int),
"offset":
validate.any(float, int),
"id":
int,
"parts": [{
"duration": validate.any(float, int),
"id": int,
"offset": validate.any(float, int),
validate.optional("recording"): int,
validate.optional("start"): validate.any(float, int)
}]
}]
}))
Chunk = namedtuple("Chunk", "recording quality sequence extension")
class BeatFLVTagConcat(FLVTagConcat):
def __init__(self, *args, **kwargs):
FLVTagConcat.__init__(self, *args, **kwargs)
def decrypt_data(self, key, iv, data):
decryptor = crypto_AES.new(key, crypto_AES.MODE_CBC, iv)
return decryptor.decrypt(data)
QUALITYS = ["original", "hd", "sd"]
QUALITY_WEIGHTS = {
"original": 1080,
"hd": 720,
"sd": 480
}
_url_re = re.compile(r"http(s)?://(?P<cdn>\w+\.)?afreeca(tv)?\.com/(?P<username>\w+)(/\d+)?")
_channel_schema = validate.Schema(
{
"CHANNEL": {
"RESULT": validate.transform(int),
validate.optional("BPWD"): validate.text,
validate.optional("BNO"): validate.text,
validate.optional("RMD"): validate.text,
validate.optional("AID"): validate.text,
validate.optional("CDN"): validate.text
}
},
validate.get("CHANNEL")
)
_stream_schema = validate.Schema(
{
validate.optional("view_url"): validate.url(
scheme=validate.any("rtmp", "http")
),
"stream_status": validate.text
class IDF1(Plugin):
DACAST_API_URL = 'https://json.dacast.com/b/{}/{}/{}'
DACAST_TOKEN_URL = 'https://services.dacast.com/token/i/b/{}/{}/{}'
_url_re = re.compile(
r'http://www\.idf1\.fr/(videos/[^/]+/[^/]+\.html|live\b)')
_video_id_re = re.compile(
r"dacast\('(?P<broadcaster_id>\d+)_(?P<video_type>[a-z]+)_(?P<video_id>\d+)', 'replay_content', data\);"
)
_video_id_alt_re = re.compile(
r'<script src="//player.dacast.com/js/player.js" id="(?P<broadcaster_id>\d+)_(?P<video_type>[cf])_(?P<video_id>\d+)"'
)
_player_url = 'http://ssl.p.jwpcdn.com/player/v/7.12.6/jwplayer.flash.swf'
_api_schema = validate.Schema(
validate.transform(parse_json), {
validate.optional('html5'): validate.all([
{
'src': validate.url()
},
], ),
'hls': validate.url(),
'hds': validate.url()
},
validate.transform(
lambda x: [update_scheme(IDF1.DACAST_API_URL, x['hls']), x['hds']
] + [y['src'] for y in x.get('html5', [])]))
_token_schema = validate.Schema(validate.transform(parse_json),
{'token': validate.text},
validate.get('token'))
_user_agent = useragents.IE_11
@classmethod
def can_handle_url(cls, url):
return IDF1._url_re.match(url)
def _get_streams(self):
res = http.get(self.url)
match = self._video_id_re.search(
res.text) or self._video_id_alt_re.search(res.text)
if match is None:
return
broadcaster_id = match.group('broadcaster_id')
video_type = match.group('video_type')
video_id = match.group('video_id')
videos = http.get(self.DACAST_API_URL.format(broadcaster_id,
video_type, video_id),
schema=self._api_schema)
token = http.get(self.DACAST_TOKEN_URL.format(broadcaster_id,
video_type, video_id),
schema=self._token_schema)
parsed = []
for video_url in videos:
video_url += token
# Ignore duplicate video URLs
if video_url in parsed:
continue
parsed.append(video_url)
# Ignore HDS streams (broken)
if '.m3u8' in video_url:
for s in HLSStream.parse_variant_playlist(
self.session, video_url).items():
yield s
_plu_schema = validate.Schema({
"playLines": [{
"urls": [{
"securityUrl":
validate.url(scheme=validate.any("rtmp", "http")),
"resolution":
validate.text,
"ext":
validate.text
}]
}]
})
_qq_schema = validate.Schema(
{validate.optional("playurl"): validate.url(scheme="http")},
validate.get("playurl"))
STREAM_WEIGHTS = {"middle": 540, "source": 1080}
class Tga(Plugin):
@classmethod
def can_handle_url(self, url):
return _url_re.match(url)
@classmethod
def stream_weight(cls, stream):
if stream in STREAM_WEIGHTS:
return STREAM_WEIGHTS[stream], "tga"
class TV8cat(Plugin):
url_re = re.compile(r"https?://(?:www\.)?tv8\.cat/directe/?")
live_iframe = "http://www.8tv.cat/wp-content/themes/8tv/_/inc/_live_html.php"
iframe_re = re.compile(r'iframe .*?src="((?:https?)?//[^"]*?)"')
account_id_re = re.compile(r"accountId:\"(\d+?)\"")
policy_key_re = re.compile(r"policyKey:\"(.+?)\"")
britecove = "https://edge.api.brightcove.com/playback/v1/accounts/{account_id}/videos/{video_id}"
britecove_schema = validate.Schema({
"sources": [{
"height": int,
validate.optional("src"): validate.url(),
validate.optional("app_name"): validate.url(scheme="rtmp"),
validate.optional("stream_name"): validate.text
}]
})
@classmethod
def can_handle_url(cls, url):
return cls.url_re.match(url) is not None
def _find_iframe(self, res):
iframe = self.iframe_re.search(res.text)
url = iframe and iframe.group(1)
if url and url.startswith("//"):
p = urlparse(self.url)
url = "{0}:{1}".format(p.scheme, url)
return url
def _britecove_params(self, url):
res = http.get(url,
headers={
"User-Agent": useragents.FIREFOX,
"Referer": self.url
})
acc = self.account_id_re.search(res.text)
pk = self.policy_key_re.search(res.text)
query = dict(parse_qsl(urlparse(url).query))
return {
"video_id": query.get("videoId"),
"account_id": acc and acc.group(1),
"policy_key": pk and pk.group(1),
}
def _get_stream_data(self, **params):
api_url = self.britecove.format(**params)
res = http.get(api_url,
headers={
"Accept":
"application/json;pk={policy_key}".format(**params)
})
return parse_json(res.text, schema=self.britecove_schema)
def _get_streams(self):
res = http.get(self.live_iframe)
britecove_url = self._find_iframe(res)
if britecove_url:
self.logger.debug("Found britecove embed url: {0}", britecove_url)
params = self._britecove_params(britecove_url)
self.logger.debug("Got britecode params: {0}", params)
stream_info = self._get_stream_data(**params)
for source in stream_info.get("sources"):
if source.get("src"):
for s in HLSStream.parse_variant_playlist(
self.session, source.get("src")).items():
yield s
else:
q = "{0}p".format(source.get("height"))
s = RTMPStream(
self.session, {
"rtmp": source.get("app_name"),
"playpath": source.get("stream_name")
})
yield q, s
from livecli.plugin.api import validate
from livecli.plugin.api.utils import parse_json
__all__ = ["parse_playlist"]
_playlist_re = re.compile(r"\(?\{.*playlist: (\[.*\]),.*?\}\)?;", re.DOTALL)
_js_to_json = partial(re.compile(r"(\w+):\s").sub, r'"\1":')
_playlist_schema = validate.Schema(
validate.transform(_playlist_re.search),
validate.any(
None,
validate.all(
validate.get(1),
validate.transform(_js_to_json),
validate.transform(parse_json),
[{
"sources": [{
"file": validate.text,
validate.optional("label"): validate.text
}]
}]
)
)
)
def parse_playlist(res):
"""Attempts to parse a JWPlayer playlist in a HTTP response body."""
return _playlist_schema.validate(res.text)
int(ts[-6:-5] + "1"))
_url_re = re.compile(
r"""
http(s)?://(\w+\.)?crunchyroll\.
(?:
com|de|es|fr|co.jp
)
(?:/[^/&?]+)?
/[^/&?]+-(?P<media_id>\d+)
""", re.VERBOSE)
_api_schema = validate.Schema({
"error": bool,
validate.optional("code"): validate.text,
validate.optional("message"): validate.text,
validate.optional("data"): object,
})
_media_schema = validate.Schema(
{
"stream_data":
validate.any(
None, {
"streams":
validate.all([{
"quality":
validate.any(validate.text, None),
"url":
validate.url(scheme="http",
path=validate.endswith(".m3u8")),
class TVPlayer(Plugin):
context_url = "http://tvplayer.com/watch/context"
api_url = "http://api.tvplayer.com/api/v2/stream/live"
login_url = "https://tvplayer.com/account/login"
update_url = "https://tvplayer.com/account/update-detail"
dummy_postcode = "SE1 9LT" # location of ITV HQ in London
url_re = re.compile(
r"https?://(?:www.)?tvplayer.com/(:?watch/?|watch/(.+)?)")
stream_attrs_re = re.compile(
r'data-(resource|token|channel-id)\s*=\s*"(.*?)"', re.S)
data_id_re = re.compile(r'data-id\s*=\s*"(.*?)"', re.S)
login_token_re = re.compile(r'input.*?name="token".*?value="(\w+)"')
stream_schema = validate.Schema(
{
"tvplayer":
validate.Schema({
"status":
u'200 OK',
"response":
validate.Schema({
"stream":
validate.url(scheme=validate.any("http", "https")),
validate.optional("drmToken"):
validate.any(None, validate.text)
})
})
}, validate.get("tvplayer"), validate.get("response"))
context_schema = validate.Schema({
"validate": validate.text,
validate.optional("token"): validate.text,
"platform": {
"key": validate.text
}
})
options = PluginOptions({"email": None, "password": None})
@classmethod
def can_handle_url(cls, url):
match = TVPlayer.url_re.match(url)
return match is not None
def __init__(self, url):
super(TVPlayer, self).__init__(url)
http.headers.update({"User-Agent": useragents.CHROME})
def authenticate(self, username, password):
res = http.get(self.login_url)
match = self.login_token_re.search(res.text)
token = match and match.group(1)
res2 = http.post(self.login_url,
data=dict(email=username,
password=password,
token=token),
allow_redirects=False)
# there is a 302 redirect on a successful login
return res2.status_code == 302
def _get_stream_data(self, resource, channel_id, token, service=1):
# Get the context info (validation token and platform)
self.logger.debug(
"Getting stream information for resource={0}".format(resource))
context_res = http.get(self.context_url,
params={
"resource": resource,
"gen": token
})
context_data = http.json(context_res, schema=self.context_schema)
self.logger.debug("Context data: {0}", str(context_data))
# get the stream urls
res = http.post(self.api_url,
data=dict(service=service,
id=channel_id,
validate=context_data["validate"],
token=context_data.get("token"),
platform=context_data["platform"]["key"]),
raise_for_status=False)
return http.json(res, schema=self.stream_schema)
def _get_stream_attrs(self, page):
stream_attrs = dict(
(k.replace("-", "_"), v.strip('"'))
for k, v in self.stream_attrs_re.findall(page.text))
if not stream_attrs.get("channel_id"):
m = self.data_id_re.search(page.text)
stream_attrs["channel_id"] = m and m.group(1)
self.logger.debug("Got stream attributes: {0}", str(stream_attrs))
valid = True
for a in ("channel_id", "resource", "token"):
if a not in stream_attrs:
self.logger.debug("Missing '{0}' from stream attributes", a)
valid = False
return stream_attrs if valid else {}
def _get_streams(self):
if self.get_option("email") and self.get_option("password"):
if not self.authenticate(self.get_option("email"),
self.get_option("password")):
self.logger.warning("Failed to login as {0}".format(
self.get_option("email")))
# find the list of channels from the html in the page
self.url = self.url.replace("https", "http") # https redirects to http
res = http.get(self.url)
if "enter your postcode" in res.text:
self.logger.info(
"Setting your postcode to: {0}. "
"This can be changed in the settings on tvplayer.com",
self.dummy_postcode)
res = http.post(self.update_url,
data=dict(postcode=self.dummy_postcode),
params=dict(return_url=self.url))
stream_attrs = self._get_stream_attrs(res)
if stream_attrs:
stream_data = self._get_stream_data(**stream_attrs)
if stream_data:
if stream_data.get("drmToken"):
self.logger.error(
"This stream is protected by DRM can cannot be played")
return
else:
return HLSStream.parse_variant_playlist(
self.session, stream_data["stream"])
else:
if "need to login" in res.text:
self.logger.error(
"You need to login using --tvplayer-email/--tvplayer-password to view this stream"
)
class Looch(Plugin):
url_re = re.compile(
r"https?://(?:www\.)?looch\.tv/channel/(?P<name>[^/]+)(/videos/(?P<video_id>\d+))?"
)
api_base = "https://api.looch.tv"
channel_api = api_base + "/channels/{name}"
video_api = api_base + "/videos/{id}"
playback_schema = validate.Schema({"weight": int, "uri": validate.url()})
data_schema = validate.Schema({
"type": validate.text,
"attributes": {
validate.optional("playback"): [playback_schema],
validate.optional("resolution"): {
"width": int,
"height": int
}
}
})
channel_schema = validate.Schema(
validate.transform(parse_json), {
"included":
validate.all(
[data_schema],
validate.filter(lambda x: x["type"] == "active_streams"),
validate.map(lambda x: x["attributes"].get("playback")),
validate.transform(lambda x: list(itertools.chain(*x)))),
}, validate.get("included"))
video_schema = validate.Schema(validate.transform(parse_json),
{"data": data_schema}, validate.get("data"),
validate.get("attributes"))
@classmethod
def can_handle_url(cls, url):
return cls.url_re.match(url) is not None
def _get_live_stream(self, channel):
url = self.channel_api.format(name=channel)
self.logger.debug("Channel API call: {0}", url)
data = http.get(url, schema=self.channel_schema)
self.logger.debug("Got {0} channel playback items", len(data))
for playback in data:
for s in HLSStream.parse_variant_playlist(self.session,
playback["uri"]).items():
yield s
def _get_video_stream(self, video_id):
url = self.video_api.format(id=video_id)
self.logger.debug("Video API call: {0}", url)
data = http.get(url, schema=self.video_schema)
self.logger.debug("Got video {0} playback items",
len(data["playback"]))
res = data["resolution"]["height"]
for playback in data["playback"]:
yield "{0}p".format(res), HTTPStream(self.session, playback["uri"])
def _get_streams(self):
match = self.url_re.match(self.url)
self.logger.debug("Matched URL: name={name}, video_id={video_id}",
**match.groupdict())
if match.group("video_id"):
return self._get_video_stream(match.group("video_id"))
elif match.group("name"):
return self._get_live_stream(match.group("name"))
r"http(s)?://(\w+\.)?weeb.tv/(channel|online)/(?P<channel>[^/&?]+)")
_schema = validate.Schema(
dict, validate.map(lambda k, v: (PARAMS_KEY_MAP.get(k, k), v)),
validate.any(
{
"status":
validate.transform(int),
"rtmp":
validate.url(scheme="rtmp"),
"playpath":
validate.text,
"multibitrate":
validate.all(validate.transform(int), validate.transform(bool)),
"block_type":
validate.transform(int),
validate.optional("token"):
validate.text,
validate.optional("block_time"):
validate.text,
validate.optional("reconnect_time"):
validate.text,
},
{
"status": validate.transform(int),
},
))
class Weeb(Plugin):
@classmethod
def can_handle_url(self, url):
def parse_fmt_list(formatsmap):
formats = {}
if not formatsmap:
return formats
for format in formatsmap.split(","):
s = format.split("/")
(w, h) = s[1].split("x")
formats[int(s[0])] = "{0}p".format(h)
return formats
_config_schema = validate.Schema({
validate.optional("fmt_list"):
validate.all(validate.text, validate.transform(parse_fmt_list)),
validate.optional("url_encoded_fmt_stream_map"):
validate.all(validate.text, validate.transform(parse_stream_map), [{
"itag":
validate.all(validate.text, validate.transform(int)),
"quality":
validate.text,
"url":
validate.url(scheme="http"),
validate.optional("s"):
validate.text,
validate.optional("stereo3d"):
validate.all(validate.text, validate.transform(int),
validate.transform(bool)),
}]),
"notes": "",
"live": True,
"vod": False,
"last_update": "2016-11-21",
}
_url_re = re.compile(r"http(s)?://(www\.)?livestream.com/")
_stream_config_schema = validate.Schema({
"event": {
"stream_info": validate.any({
"is_live": bool,
"qualities": [{
"bitrate": int,
"height": int
}],
validate.optional("play_url"): validate.url(scheme="http"),
validate.optional("m3u8_url"): validate.url(
scheme="http",
path=validate.endswith(".m3u8")
),
}, None)
},
validate.optional("playerUri"): validate.text,
validate.optional("viewerPlusSwfUrl"): validate.url(scheme="http"),
validate.optional("lsPlayerSwfUrl"): validate.text,
validate.optional("hdPlayerSwfUrl"): validate.text
})
_smil_schema = validate.Schema(validate.union({
"http_base": validate.all(
validate.xml_find("{http://www.w3.org/2001/SMIL20/Language}head/"
"{http://www.w3.org/2001/SMIL20/Language}meta"
__livecli_docs__ = {
"domains": [
"camsoda.com",
],
"geo_blocked": [],
"notes": "",
"live": True,
"vod": False,
"last_update": "2017-11-18",
}
_url_re = re.compile(r"http(s)?://(www\.)?camsoda\.com/(?P<username>[^\"\']+)")
_api_user_schema = validate.Schema({
"status": validate.any(int, validate.text),
validate.optional("user"): {
"online": validate.any(int, validate.text),
"chatstatus": validate.text,
}
})
_api_video_schema = validate.Schema({
"token": validate.text,
"app": validate.text,
"edge_servers": [validate.text],
"stream_name": validate.text
})
class Camsoda(Plugin):
API_URL_USER = "******"
],
"geo_blocked": [],
"notes": "",
"live": True,
"vod": True,
"last_update": "2014-06-22",
}
API_CLIENT_NAME = "Bambuser AS2"
API_CONTEXT = "b_broadcastpage"
API_KEY = "005f64509e19a868399060af746a00aa"
API_URL_VIDEO = "http://player-c.api.bambuser.com/getVideo.json"
_url_re = re.compile(r"http(s)?://(\w+.)?bambuser.com/v/(?P<video_id>\d+)")
_video_schema = validate.Schema({
validate.optional("error"): validate.text,
validate.optional("result"): {
"id": validate.text,
"size": validate.text,
"url": validate.url(scheme=validate.any("rtmp", "http"))
}
})
class Bambuser(Plugin):
@classmethod
def can_handle_url(self, url):
return _url_re.match(url)
def _get_streams(self):
match = _url_re.match(self.url)
