def __update_m3u8(self, url, part, headers, use_kodi_hls):
""" Update a video that has M3u8 streams.
:param str url: The URL for the stream.
:param MediaItemPart part: The new part that needs updating.
:param dict[str,str] headers: The URL headers to use.
:param bool use_kodi_hls: Should we use the InputStream Adaptive add-on?
"""
# first see if there are streams in this file, else check the second location.
for s, b in M3u8.get_streams_from_m3u8(url,
self.proxy,
headers=headers):
if use_kodi_hls:
strm = part.append_media_stream(url, 0)
M3u8.set_input_stream_addon_input(strm, headers=headers)
# Only the main M3u8 is needed
break
else:
part.append_media_stream(s, b)
if not part.MediaStreams and "manifest.m3u8" in url:
Logger.warning(
"No streams found in %s, trying alternative with 'master.m3u8'",
url)
url = url.replace("manifest.m3u8", "master.m3u8")
for s, b in M3u8.get_streams_from_m3u8(url,
self.proxy,
headers=headers):
if use_kodi_hls:
strm = part.append_media_stream(url, 0)
M3u8.set_input_stream_addon_input(strm, headers=headers)
# Only the main M3u8 is needed
break
else:
part.append_media_stream(s, b)
# check for subs
# https://mtgxse01-vh.akamaihd.net/i/201703/13/DCjOLN_1489416462884_427ff3d3_,48,260,460,900,1800,2800,.mp4.csmil/master.m3u8?__b__=300&hdnts=st=1489687185~exp=3637170832~acl=/*~hmac=d0e12e62c219d96798e5b5ef31b11fa848724516b255897efe9808c8a499308b&cc1=name=Svenska%20f%C3%B6r%20h%C3%B6rselskadade~default=no~forced=no~lang=sv~uri=https%3A%2F%2Fsubstitch.play.mtgx.tv%2Fsubtitle%2Fconvert%2Fxml%3Fsource%3Dhttps%3A%2F%2Fcdn-subtitles-mtgx-tv.akamaized.net%2Fpitcher%2F20xxxxxx%2F2039xxxx%2F203969xx%2F20396967%2F20396967-swt.xml%26output%3Dm3u8
# https://cdn-subtitles-mtgx-tv.akamaized.net/pitcher/20xxxxxx/2039xxxx/203969xx/20396967/20396967-swt.xml&output=m3u8
if "uri=" in url and not part.Subtitle:
Logger.debug("Extracting subs from M3u8")
sub_url = url.rsplit("uri=")[-1]
sub_url = HtmlEntityHelper.url_decode(sub_url)
sub_data = UriHandler.open(sub_url, proxy=self.proxy)
subs = [
line for line in sub_data.split("\n")
if line.startswith("http")
]
if subs:
part.Subtitle = SubtitleHelper.download_subtitle(
subs[0], format='webvtt', proxy=self.proxy)
return
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
json = JsonHelper(data)
video_info = json.get_value("content", "videoInfos")
part = item.create_new_empty_media_part()
if "HLSurlHD" in video_info:
# HLSurlHD=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/
# potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,q50,q60,.mp4.csmil/master.m3u8
for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurlHD"],
self.proxy):
item.complete = True
part.append_media_stream(s, b)
elif "HLSurl" in video_info:
# HLSurl=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/
# potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,.mp4.csmil/master.m3u8
for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurl"],
self.proxy):
item.complete = True
part.append_media_stream(s, b)
if "downloadLink" in video_info:
# downloadLink=http://podcastsource.sf.tv/nps/podcast/10vor10/2015/03/
# 10vor10_20150304_215030_v_podcast_h264_q10.mp4
part.append_media_stream(video_info["downloadLink"], 1000)
return item
def __update_live_video(self, item, manifest, headers):
video_info = manifest.get_value("playable", "assets", 0)
url = video_info["url"]
encrypted = video_info["encrypted"]
part = item.create_new_empty_media_part()
if encrypted:
use_adaptive = AddonSettings.use_adaptive_stream_add_on(with_encryption=True)
if not use_adaptive:
Logger.error("Cannot playback encrypted item without inputstream.adaptive with encryption support")
return item
stream = part.append_media_stream(url, 0)
key = M3u8.get_license_key("", key_headers=headers, key_type="R")
M3u8.set_input_stream_addon_input(stream, proxy=self.proxy, headers=headers, license_key=key)
item.complete = True
else:
use_adaptive = AddonSettings.use_adaptive_stream_add_on(with_encryption=False)
if use_adaptive:
stream = part.append_media_stream(url, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy, headers=headers)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy, headers=headers):
item.complete = True
part.append_media_stream(s, b)
return item
def update_live_stream(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
part = item.create_new_empty_media_part()
if item.url == "#livetv":
url = "https://d34pj260kw1xmk.cloudfront.net/live/l1/tv/index.m3u8"
M3u8.update_part_with_m3u8_streams(part, url, encrypted=True)
else:
# the audio won't play with the InputStream Adaptive add-on.
url = "https://d34pj260kw1xmk.cloudfront.net/live/l1/radio/index.m3u8"
for s, b in M3u8.get_streams_from_m3u8(url):
part.append_media_stream(s, b)
item.complete = True
return item
def update_live_item(self, item):
""" Updates an existing live MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
part = item.create_new_empty_media_part()
if AddonSettings.use_adaptive_stream_add_on():
stream = part.append_media_stream(item.url, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(item.url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def __update_video_from_brightcove(self, item, data,
use_adaptive_with_encryption):
""" Updates an existing MediaItem with more data based on an MPD stream.
:param str data: Stream info retrieved from BrightCove.
:param bool use_adaptive_with_encryption: Do we use the Adaptive InputStream add-on?
:param MediaItem item: The original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
part = item.create_new_empty_media_part()
# Then try the new BrightCove JSON
bright_cove_regex = '<video[^>]+data-video-id="(?<videoId>[^"]+)[^>]+data-account="(?<videoAccount>[^"]+)'
bright_cove_data = Regexer.do_regex(
Regexer.from_expresso(bright_cove_regex), data)
if not bright_cove_data:
Logger.warning("Error updating using BrightCove data: %s", item)
return item
Logger.info("Found new BrightCove JSON data")
bright_cove_url = 'https://edge.api.brightcove.com/playback/v1/accounts/' \
'%(videoAccount)s/videos/%(videoId)s' % bright_cove_data[0]
headers = {
"Accept":
"application/json;pk=BCpkADawqM3ve1c3k3HcmzaxBvD8lXCl89K7XEHiKutxZArg2c5RhwJHJANOwPwS_4o7UsC4RhIzXG8Y69mrwKCPlRkIxNgPQVY9qG78SJ1TJop4JoDDcgdsNrg"
}
bright_cove_data = UriHandler.open(bright_cove_url,
additional_headers=headers)
bright_cove_json = JsonHelper(bright_cove_data)
streams = [
d for d in bright_cove_json.get_value("sources")
if d["container"] == "M2TS"
]
# Old filter
# streams = filter(lambda d: d["container"] == "M2TS", bright_cove_json.get_value("sources"))
if not streams:
Logger.warning("Error extracting streams from BrightCove data: %s",
item)
return item
# noinspection PyTypeChecker
stream_url = streams[0]["src"]
# these streams work better with the the InputStreamAddon because it removes the
# "range" http header
if use_adaptive_with_encryption:
Logger.info("Using InputStreamAddon for playback of HLS stream")
strm = part.append_media_stream(stream_url, 0)
M3u8.set_input_stream_addon_input(strm)
item.complete = True
return item
for s, b in M3u8.get_streams_from_m3u8(stream_url):
item.complete = True
part.append_media_stream(s, b)
return item
def update_video_item(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv
# http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url)
part = item.create_new_empty_media_part()
if "mediazone.vrt.be" not in item.url:
# Extract actual media data
video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0]
url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, )
data = UriHandler.open(url)
json = JsonHelper(data)
urls = json.get_value("targetUrls")
for url_info in urls:
Logger.trace(url_info)
if url_info["type"].lower() != "hls":
continue
hls_url = url_info["url"]
for s, b in M3u8.get_streams_from_m3u8(hls_url):
part.append_media_stream(s, b)
item.complete = True
return item
def update_json_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
headers = {}
if self.localIP:
headers.update(self.localIP)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=headers)
video_data = JsonHelper(data)
stream_data = video_data.get_value("mediaAssetsOnDemand")
if not stream_data:
return item
use_adaptive = AddonSettings.use_adaptive_stream_add_on()
stream_data = stream_data[0]
part = item.create_new_empty_media_part()
if "hlsUrl" in stream_data:
hls_url = stream_data["hlsUrl"]
if use_adaptive:
stream = part.append_media_stream(hls_url, 0)
M3u8.set_input_stream_addon_input(stream,
self.proxy,
headers=headers)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(hls_url,
self.proxy,
headers=headers):
item.complete = True
part.append_media_stream(s, b)
if "timedTextSubtitlesUrl" in stream_data and stream_data[
"timedTextSubtitlesUrl"]:
sub_url = stream_data["timedTextSubtitlesUrl"].replace(
".ttml", ".vtt")
sub_url = HtmlEntityHelper.url_decode(sub_url)
part.Subtitle = SubtitleHelper.download_subtitle(sub_url,
format="webvtt")
return item
def add_others(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
others = MediaItem(
"\b.: Populair :.",
"https://api.kijk.nl/v2/default/sections/popular_PopularVODs?offset=0"
)
items.append(others)
days = MediaItem("\b.: Deze week :.", "#lastweek")
items.append(days)
search = MediaItem("\b.: Zoeken :.", "searchSite")
search.complete = True
search.dontGroup = True
search.HttpHeaders = {"X-Requested-With": "XMLHttpRequest"}
items.append(search)
if self.channelCode == "veronica":
live = LanguageHelper.get_localized_string(
LanguageHelper.LiveStreamTitleId)
live_radio = MediaItem("Radio Veronica {}".format(live), "")
live_radio.type = "video"
live_radio.dontGroup = True
part = live_radio.create_new_empty_media_part()
live_stream = "https://talparadiohls-i.akamaihd.net/hls/live/585615/VR-Veronica-1/playlist.m3u8"
if AddonSettings.use_adaptive_stream_add_on(with_encryption=False,
channel=self):
stream = part.append_media_stream(live_stream, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy)
live_radio.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(live_stream,
self.proxy):
live_radio.complete = True
part.append_media_stream(s, b)
items.append(live_radio)
Logger.debug("Pre-Processing finished")
return data, items
def __update_live_audio(self, item, manifest, headers):
video_info = manifest.get_value("playable", "assets", 0)
url = video_info["url"]
# Is it encrypted? encrypted = video_info["encrypted"]
part = item.create_new_empty_media_part()
# Adaptive add-on does not work with audio only
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy, headers=headers):
item.complete = True
part.append_media_stream(s, b)
return item
def update_live_item(self, item):
""" Updates an existing MediaItem for a live stream with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_live_item for %s (%s)', item.name,
self.channelName)
item.MediaItemParts = []
part = item.create_new_empty_media_part()
spoof_ip = self._get_setting("spoof_ip", "0.0.0.0")
if spoof_ip:
for s, b in M3u8.get_streams_from_m3u8(
item.url, self.proxy,
headers={"X-Forwarded-For": spoof_ip}):
part.append_media_stream(s, b)
else:
for s, b in M3u8.get_streams_from_m3u8(item.url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
headers = {}
if self.localIP:
headers.update(self.localIP)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=headers)
m3u8_url = Regexer.do_regex('data-file="([^"]+)"', data)[0]
part = item.create_new_empty_media_part()
if AddonSettings.use_adaptive_stream_add_on(with_encryption=False):
stream = part.append_media_stream(m3u8_url, 0)
M3u8.set_input_stream_addon_input(stream,
proxy=self.proxy,
headers=headers)
item.complete = True
else:
for s, b, a in M3u8.get_streams_from_m3u8(m3u8_url,
self.proxy,
headers=headers,
map_audio=True):
if a and "-audio" not in s:
video_part = s.rsplit("-", 1)[-1]
video_part = "-%s" % (video_part, )
s = a.replace(".m3u8", video_part)
part.append_media_stream(s, b)
item.complete = True
return item
def __get_video_streams(self, video_id, part):
""" Fetches the video stream for a given videoId
@param video_id: (integer) the videoId
@param part: (MediaPart) the mediapart to add the streams to
@return: (bool) indicating a successfull retrieval
"""
# hardcoded for now as it does not seem top matter
dscgeo = '{"countryCode":"%s","expiry":1446917369986}' % (
self.language.upper(), )
dscgeo = HtmlEntityHelper.url_encode(dscgeo)
headers = {"Cookie": "dsc-geo=%s" % (dscgeo, )}
# send the data
http, nothing, host, other = self.baseUrl.split("/", 3)
subdomain, domain = host.split(".", 1)
url = "https://secure.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls" \
% (domain, video_id,)
data = UriHandler.open(url,
proxy=self.proxy,
additional_headers=headers,
no_cache=True)
json = JsonHelper(data)
url = json.get_value("hls")
if url is None:
return False
streams_found = False
if "?" in url:
qs = url.split("?")[-1]
else:
qs = None
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy):
# and we need to append the original QueryString
if "X-I-FRAME-STREAM" in s:
continue
streams_found = True
if qs is not None:
if "?" in s:
s = "%s&%s" % (s, qs)
else:
s = "%s?%s" % (s, qs)
part.append_media_stream(s, b)
return streams_found
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
if not item.url.endswith("m3u8"):
data = UriHandler.open(item.url, proxy=self.proxy)
json_data = Regexer.do_regex(self.mediaUrlRegex, data)
if not json_data:
Logger.error("Cannot find JSON stream info.")
return item
json = JsonHelper(json_data[0])
Logger.trace(json.json)
stream = json.get_value("source", "hls")
if stream is None:
stream = json.get_value("mzsource", "hls")
Logger.debug("Found HLS: %s", stream)
else:
stream = item.url
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(stream, self.proxy):
item.complete = True
part.append_media_stream(s, b)
# var playerConfig = {"id":"mediaplayer","width":"100%","height":"100%","autostart":"false","image":"http:\/\/www.ketnet.be\/sites\/default\/files\/thumb_5667ea22632bc.jpg","brand":"ketnet","source":{"hls":"http:\/\/vod.stream.vrt.be\/ketnet\/_definst_\/mp4:ketnet\/2015\/12\/Ben_ik_familie_van_R001_A0023_20151208_143112_864.mp4\/playlist.m3u8"},"analytics":{"type_stream":"vod","playlist":"Ben ik familie van?","program":"Ben ik familie van?","episode":"Ben ik familie van?: Warre - Aflevering 3","parts":"1","whatson":"270157835527"},"title":"Ben ik familie van?: Warre - Aflevering 3","description":"Ben ik familie van?: Warre - Aflevering 3"}
return item
def update_live_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
part = item.create_new_empty_media_part()
hls_url = media_info.get_value("streamUrl")
if hls_url is not None and "m3u8" in hls_url:
Logger.debug("Found HLS url for %s: %s",
media_info.json["streamName"], hls_url)
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
else:
Logger.debug("No HLS url found for %s. Fetching RTMP Token.",
media_info.json["streamName"])
# fetch the token:
token_url = "%s/api/media/streaming?streamname=%s" \
% (self.baseUrl, media_info.json["streamName"])
token_data = UriHandler.open(token_url,
proxy=self.proxy,
additional_headers=item.HttpHeaders,
no_cache=True)
token_data = JsonHelper(token_data)
token = token_data.get_value("token")
Logger.debug("Found token '%s' for '%s'", token,
media_info.json["streamName"])
rtmp_url = "rtmp://rtmp.rtbf.be/livecast/%s?%s pageUrl=%s tcUrl=rtmp://rtmp.rtbf.be/livecast" \
% (media_info.json["streamName"], token, self.baseUrl)
rtmp_url = self.get_verifiable_video_url(rtmp_url)
part.append_media_stream(rtmp_url, 0)
item.complete = True
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
return item
def update_live_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
# http://services.vrt.be/videoplayer/r/live.json?_1466364209811=
channel_data = UriHandler.open(
"http://services.vrt.be/videoplayer/r/live.json", proxy=self.proxy)
channel_data = JsonHelper(channel_data)
url = None
for channel_id in channel_data.json:
if channel_id not in item.url:
continue
else:
url = channel_data.json[channel_id].get("hls")
if url is None:
Logger.error("Could not find stream for live channel: %s",
item.url)
return item
Logger.debug("Found stream url for %s: %s", item, url)
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def __update_video(self, item, data):
regex = 'data-file="([^"]+)'
m3u8_url = Regexer.do_regex(regex, data)[-1]
if ".m3u8" not in m3u8_url:
Logger.info("Not a direct M3u8 file. Need to log in")
url = "https://api.viervijfzes.be/content/%s" % (m3u8_url, )
# We need to log in
if not self.loggedOn:
self.log_on()
# add authorization header
authentication_header = {
"authorization": self.__idToken,
"content-type": "application/json"
}
data = UriHandler.open(url,
proxy=self.proxy,
additional_headers=authentication_header)
json_data = JsonHelper(data)
m3u8_url = json_data.get_value("video", "S")
# Geo Locked?
if "geo" in m3u8_url.lower():
# set it for the error statistics
item.isGeoLocked = True
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(m3u8_url, self.proxy):
if int(b) < 200:
Logger.info("Skipping stream of quality '%s' kbps", b)
continue
item.complete = True
part.append_media_stream(s, b)
item.complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
if item.metaData.get(self.__REQUIRES_LOGIN, False):
logged_in = self.log_on()
if not logged_in:
XbmcWrapper.show_dialog(LanguageHelper.LoginErrorTitle,
LanguageHelper.LoginErrorText)
return item
video_data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=self.localIP)
if not video_data:
return item
video_data = JsonHelper(video_data)
video_info = video_data.get_value("data", "attributes")
errors = video_data.get_value("errors")
Logger.error("Error updating items: %s", errors)
if errors:
return item
part = item.create_new_empty_media_part()
m3u8url = video_info["streaming"]["hls"]["url"]
m3u8data = UriHandler.open(m3u8url, self.proxy)
if AddonSettings.use_adaptive_stream_add_on():
stream = part.append_media_stream(m3u8url, 0)
item.complete = True
M3u8.set_input_stream_addon_input(stream, self.proxy)
else:
# user agent for all sub m3u8 and ts requests needs to be the same
part.HttpHeaders[
"user-agent"] = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)"
for s, b, a in M3u8.get_streams_from_m3u8(
m3u8url,
self.proxy,
append_query_string=False,
map_audio=True,
play_list_data=m3u8data):
item.complete = True
if a:
audio_part = a.split("-prog_index.m3u8", 1)[0]
audio_id = audio_part.rsplit("/", 1)[-1]
s = s.replace("-prog_index.m3u8",
"-{0}-prog_index.m3u8".format(audio_id))
part.append_media_stream(s, b)
if self.language == "se":
vtt_url = M3u8.get_subtitle(m3u8url,
self.proxy,
m3u8data,
language="sv")
elif self.language == "dk":
vtt_url = M3u8.get_subtitle(m3u8url,
self.proxy,
m3u8data,
language="da")
else:
vtt_url = M3u8.get_subtitle(m3u8url, self.proxy, m3u8data)
# https://dplaynordics-vod-80.akamaized.net/dplaydni/259/0/hls/243241001/1112635959-prog_index.m3u8?version_hash=bb753129&hdnts=st=1518218118~exp=1518304518~acl=/*~hmac=bdeefe0ec880f8614e14af4d4a5ca4d3260bf2eaa8559e1eb8ba788645f2087a
vtt_url = vtt_url.replace("-prog_index.m3u8", "-0.vtt")
part.Subtitle = SubtitleHelper.download_subtitle(vtt_url,
format='srt',
proxy=self.proxy)
# if the user has premium, don't show any warnings
if self.__has_premium:
item.isPaid = False
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# noinspection PyStatementEffect
"""
C:\temp\rtmpdump-2.3>rtmpdump.exe -z -o test.flv -n "cp70051.edgefcs.net" -a "tv
4ondemand" -y "mp4:/mp4root/2010-06-02/pid2780626_1019976_T3MP48_.mp4?token=c3Rh
cnRfdGltZT0yMDEwMDcyNjE2NDYyNiZlbmRfdGltZT0yMDEwMDcyNjE2NDgyNiZkaWdlc3Q9ZjFjN2U1
NTRiY2U5ODMxMDMwYWQxZWEwNzNhZmUxNjI=" -l 2
C:\temp\rtmpdump-2.3>rtmpdump.exe -z -o test.flv -r rtmpe://cp70051.edgefcs.net/
tv4ondemand/mp4root/2010-06-02/pid2780626_1019976_T3MP48_.mp4?token=c3RhcnRfdGlt
ZT0yMDEwMDcyNjE2NDYyNiZlbmRfdGltZT0yMDEwMDcyNjE2NDgyNiZkaWdlc3Q9ZjFjN2U1NTRiY2U5
ODMxMDMwYWQxZWEwNzNhZmUxNjI=
"""
# retrieve the mediaurl
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=self.localIP)
stream_info = JsonHelper(data)
stream_url = stream_info.get_value("playbackItem", "manifestUrl")
if stream_url is None:
return item
if ".mpd" in stream_url:
return self.__update_dash_video(item, stream_info)
part = item.create_new_empty_media_part()
if AddonSettings.use_adaptive_stream_add_on() and False:
subtitle = M3u8.get_subtitle(stream_url, proxy=self.proxy)
stream = part.append_media_stream(stream_url, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy)
item.complete = True
else:
m3u8_data = UriHandler.open(stream_url,
proxy=self.proxy,
additional_headers=self.localIP)
subtitle = M3u8.get_subtitle(stream_url,
proxy=self.proxy,
play_list_data=m3u8_data)
for s, b, a in M3u8.get_streams_from_m3u8(stream_url,
self.proxy,
play_list_data=m3u8_data,
map_audio=True):
item.complete = True
if not item.isLive and "-video" not in s:
continue
if a and "-audio" not in s:
# remove any query parameters
video_part = s.rsplit("?", 1)[0]
video_part = video_part.rsplit("-", 1)[-1]
video_part = "-%s" % (video_part, )
s = a.replace(".m3u8", video_part)
part.append_media_stream(s, b)
if subtitle:
subtitle = subtitle.replace(".m3u8", ".webvtt")
part.Subtitle = SubtitleHelper.download_subtitle(subtitle,
format="m3u8srt",
proxy=self.proxy)
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# noinspection PyStatementEffect
"""
data-video-id="1613274"
data-video-type="video"
data-video-src="http://media.vrtnieuws.net/2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-title="Het journaal 1 - 25/04/13"
data-video-rtmp-server="rtmp://vrt.flash.streampower.be/vrtnieuws"
data-video-rtmp-path="2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-rtmpt-server="rtmpt://vrt.flash.streampower.be/vrtnieuws"
data-video-rtmpt-path="2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-iphone-server="http://iphone.streampower.be/vrtnieuws_nogeo/_definst_"
data-video-iphone-path="2013/04/135132051ONL1304255866693.urlMP4_H.264.m4v"
data-video-mobile-server="rtsp://mp4.streampower.be/vrt/vrt_mobile/vrtnieuws_nogeo"
data-video-mobile-path="2013/04/135132051ONL1304255866693.url3GP_MPEG4.3gp"
data-video-sitestat-program="het_journaal_1_-_250413_id_1-1613274"
"""
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url, proxy=self.proxy)
data = data.replace("\\/", "/")
urls = Regexer.do_regex(self.mediaUrlRegex, data)
part = item.create_new_empty_media_part()
for url in urls:
Logger.trace(url)
if url[0] == "src":
flv = url[1]
bitrate = 750
else:
flv_server = url[1]
flv_path = url[2]
if url[0] == "rtmp-server":
flv = "%s//%s" % (flv_server, flv_path)
bitrate = 750
elif url[0] == "rtmpt-server":
continue
# Not working for now
#flv = "%s//%s" % (flv_server, flv_path)
#flv = self.get_verifiable_video_url(flv)
#bitrate = 1500
elif url[0] == "iphone-server":
flv = "%s/%s" % (flv_server, flv_path)
if not flv.endswith("playlist.m3u8"):
flv = "%s/playlist.m3u8" % (flv,)
for s, b in M3u8.get_streams_from_m3u8(flv, self.proxy):
item.complete = True
part.append_media_stream(s, b)
# no need to continue adding the streams
continue
elif url[0] == "mobile-server":
flv = "%s/%s" % (flv_server, flv_path)
bitrate = 250
else:
flv = "%s/%s" % (flv_server, flv_path)
bitrate = 0
part.append_media_stream(flv, bitrate)
item.complete = True
return item
def get_stream_info(self, item, mzid, live=False, hls_over_dash=False): # NOSONAR
""" Updates an item with Vualto stream data.
:param MediaItem item: The Mediaitem to update
:param str mzid: The MZ ID of the stream
:param bool live: Indicator if the stream is live or not
:param bool hls_over_dash: Should we prefer HLS over Dash?
:return: An updated MediaItem
:rtype: MediaItem
"""
# We need a player token
token_data = UriHandler.open("https://media-services-public.vrt.be/"
"vualto-video-aggregator-web/rest/external/v1/tokens", data="",
additional_headers={"Content-Type": "application/json"})
token = JsonHelper(token_data).get_value("vrtPlayerToken")
asset_url = "https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/" \
"external/v1/videos/{0}?vrtPlayerToken={1}&client={2}" \
.format(mzid, HtmlEntityHelper.url_encode(token), self.client_id)
asset_data = UriHandler.open(asset_url, no_cache=True)
asset_data = JsonHelper(asset_data)
drm_key = asset_data.get_value("drm")
drm_protected = drm_key is not None
adaptive_available = AddonSettings.use_adaptive_stream_add_on(
with_encryption=drm_protected, channel=self.channel)
part = item.create_new_empty_media_part()
srt = None
# see if we prefer hls over dash
hls_prio = 2 if hls_over_dash else 0
for target_url in asset_data.get_value("targetUrls"):
video_type = target_url["type"]
video_url = target_url["url"]
if video_type == "hls_aes" and drm_protected and adaptive_available:
# no difference in encrypted or not.
Logger.debug("Found HLS AES encrypted stream and a DRM key")
stream = part.append_media_stream(video_url, hls_prio)
M3u8.set_input_stream_addon_input(stream)
elif video_type == "hls" and not drm_protected:
# no difference in encrypted or not.
if adaptive_available:
Logger.debug("Found standard HLS stream and without DRM protection")
stream = part.append_media_stream(video_url, hls_prio)
M3u8.set_input_stream_addon_input(stream)
else:
m3u8_data = UriHandler.open(video_url)
for s, b, a in M3u8.get_streams_from_m3u8(video_url,
play_list_data=m3u8_data,
map_audio=True):
item.complete = True
if a:
audio_part = a.rsplit("-", 1)[-1]
audio_part = "-%s" % (audio_part,)
s = s.replace(".m3u8", audio_part)
part.append_media_stream(s, b)
srt = M3u8.get_subtitle(video_url, play_list_data=m3u8_data)
if not srt or live:
# If there is not SRT don't download it. If it a live stream with subs,
# don't use it as it is not supported by Kodi
continue
srt = srt.replace(".m3u8", ".vtt")
part.Subtitle = SubtitleHelper.download_subtitle(srt, format="webvtt")
elif video_type == "mpeg_dash" and adaptive_available:
if not drm_protected:
Logger.debug("Found standard MPD stream and without DRM protection")
stream = part.append_media_stream(video_url, 1)
Mpd.set_input_stream_addon_input(stream)
else:
stream = part.append_media_stream(video_url, 1)
encryption_json = '{{"token":"{0}","drm_info":[D{{SSM}}],"kid":"{{KID}}"}}' \
.format(drm_key)
encryption_key = Mpd.get_license_key(
key_url="https://widevine-proxy.drm.technology/proxy",
key_type="D",
key_value=encryption_json,
key_headers={"Content-Type": "text/plain;charset=UTF-8"}
)
Mpd.set_input_stream_addon_input(stream, license_key=encryption_key)
if video_type.startswith("hls") and srt is None:
srt = M3u8.get_subtitle(video_url)
if not srt or live:
# If there is not SRT don't download it. If it a live stream with subs,
# don't use it as it is not supported by Kodi
continue
srt = srt.replace(".m3u8", ".vtt")
part.Subtitle = SubtitleHelper.download_subtitle(srt, format="webvtt")
item.complete = True
return item
def get_streams_from_npo(url, episode_id, proxy=None, headers=None):
""" Retrieve NPO Player Live streams from a different number of stream urls.
@param url: (String) The url to download
@param episode_id: (String) The NPO episode ID
@param headers: (dict) Possible HTTP Headers
@param proxy: (Proxy) The proxy to use for opening
Can be used like this:
part = item.create_new_empty_media_part()
for s, b in NpoStream.get_streams_from_npo(m3u8Url, self.proxy):
item.complete = True
# s = self.get_verifiable_video_url(s)
part.append_media_stream(s, b)
"""
if url:
Logger.info("Determining streams for url: %s", url)
episode_id = url.split("/")[-1]
elif episode_id:
Logger.info("Determining streams for VideoId: %s", episode_id)
else:
Logger.error("No url or streamId specified!")
return []
# we need an hash code
token_json_data = UriHandler.open("http://ida.omroep.nl/app.php/auth",
no_cache=True,
proxy=proxy,
additional_headers=headers)
token_json = JsonHelper(token_json_data)
token = token_json.get_value("token")
url = "http://ida.omroep.nl/app.php/%s?adaptive=yes&token=%s" % (
episode_id, token)
stream_data = UriHandler.open(url,
proxy=proxy,
additional_headers=headers)
if not stream_data:
return []
stream_json = JsonHelper(stream_data, logger=Logger.instance())
stream_infos = stream_json.get_value("items")[0]
Logger.trace(stream_infos)
streams = []
for stream_info in stream_infos:
Logger.debug("Found stream info: %s", stream_info)
if stream_info["format"] == "mp3":
streams.append((stream_info["url"], 0))
continue
elif stream_info["contentType"] == "live":
Logger.debug("Found live stream")
url = stream_info["url"]
url = url.replace("jsonp", "json")
live_url_data = UriHandler.open(url,
proxy=proxy,
additional_headers=headers)
live_url = live_url_data.strip("\"").replace("\\", "")
Logger.trace(live_url)
streams += M3u8.get_streams_from_m3u8(live_url,
proxy,
headers=headers)
elif stream_info["format"] == "hls":
m3u8_info_url = stream_info["url"]
m3u8_info_data = UriHandler.open(m3u8_info_url,
proxy=proxy,
additional_headers=headers)
m3u8_info_json = JsonHelper(m3u8_info_data,
logger=Logger.instance())
m3u8_url = m3u8_info_json.get_value("url")
streams += M3u8.get_streams_from_m3u8(m3u8_url,
proxy,
headers=headers)
elif stream_info["format"] == "mp4":
bitrates = {"hoog": 1000, "normaal": 500}
url = stream_info["url"]
if "contentType" in stream_info and stream_info[
"contentType"] == "url":
mp4_url = url
else:
url = url.replace("jsonp", "json")
mp4_url_data = UriHandler.open(url,
proxy=proxy,
additional_headers=headers)
mp4_info_json = JsonHelper(mp4_url_data,
logger=Logger.instance())
mp4_url = mp4_info_json.get_value("url")
bitrate = bitrates.get(stream_info["label"].lower(), 0)
if bitrate == 0 and "/ipod/" in mp4_url:
bitrate = 200
elif bitrate == 0 and "/mp4/" in mp4_url:
bitrate = 500
streams.append((mp4_url, bitrate))
return streams
def update_json_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
headers = {"accept": "application/vnd.sbs.ovp+json; version=2.0"}
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=headers)
if UriHandler.instance().status.code == 404:
Logger.warning("No normal stream found. Trying newer method")
new_url = item.url.replace("https://embed.kijk.nl/api/",
"https://embed.kijk.nl/")
item.url = new_url[:new_url.index("?")]
return self.__update_embedded_video(item)
json = JsonHelper(data)
embed_url = json.get_value("metadata", "embedURL")
if embed_url:
Logger.warning(
"Embed URL found. Using that to determine the streams.")
item.url = embed_url
try:
return self.__update_embedded_video(item)
except:
Logger.warning("Failed to update embedded item:",
exc_info=True)
use_adaptive_with_encryption = AddonSettings.use_adaptive_stream_add_on(
with_encryption=True, channel=self)
mpd_info = json.get_value("entitlements", "play")
# is there MPD information in the API response?
if mpd_info is not None:
return self.__update_video_from_mpd(item, mpd_info,
use_adaptive_with_encryption)
# Try the plain M3u8 streams
part = item.create_new_empty_media_part()
m3u8_url = json.get_value("playlist")
use_adaptive = AddonSettings.use_adaptive_stream_add_on(channel=self)
# with the Accept: application/vnd.sbs.ovp+json; version=2.0 header, the m3u8 streams that
# are brightcove based have an url parameter instead of an empty m3u8 file
Logger.debug("Trying standard M3u8 streams.")
if m3u8_url != "https://embed.kijk.nl/api/playlist/.m3u8" \
and "hostingervice=brightcove" not in m3u8_url:
for s, b in M3u8.get_streams_from_m3u8(m3u8_url,
self.proxy,
append_query_string=True):
if "_enc_" in s:
continue
if use_adaptive:
# we have at least 1 none encrypted streams
Logger.info("Using HLS InputStreamAddon")
strm = part.append_media_stream(m3u8_url, 0)
M3u8.set_input_stream_addon_input(strm, proxy=self.proxy)
item.complete = True
return item
part.append_media_stream(s, b)
item.complete = True
return item
Logger.warning("No M3u8 data found. Falling back to BrightCove")
video_id = json.get_value("vpakey")
# videoId = json.get_value("videoId") -> Not all items have a videoId
mpd_manifest_url = "https://embed.kijk.nl/video/%s?width=868&height=491" % (
video_id, )
referer = "https://embed.kijk.nl/video/%s" % (video_id, )
data = UriHandler.open(mpd_manifest_url,
proxy=self.proxy,
referer=referer)
# First try to find an M3u8
m3u8_urls = Regexer.do_regex('https:[^"]+.m3u8', data)
for m3u8_url in m3u8_urls:
m3u8_url = m3u8_url.replace("\\", "")
# We need the actual URI to make this work, so fetch it.
m3u8_url = UriHandler.header(m3u8_url, proxy=self.proxy)[-1]
Logger.debug("Found direct M3u8 in brightcove data.")
if use_adaptive:
# we have at least 1 none encrypted streams
Logger.info("Using HLS InputStreamAddon")
strm = part.append_media_stream(m3u8_url, 0)
M3u8.set_input_stream_addon_input(strm, proxy=self.proxy)
item.complete = True
return item
for s, b in M3u8.get_streams_from_m3u8(m3u8_url,
self.proxy,
append_query_string=True):
item.complete = True
part.append_media_stream(s, b)
return item
return self.__update_video_from_brightcove(
item, data, use_adaptive_with_encryption)
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
data = UriHandler.open(item.url, proxy=self.proxy)
json = JsonHelper(data, Logger.instance())
video_data = json.get_value("video")
if video_data:
part = item.create_new_empty_media_part()
if self.localIP:
part.HttpHeaders.update(self.localIP)
# get the videos
video_urls = video_data.get("videoReferences")
for video_url in video_urls:
# Logger.Trace(videoUrl)
stream_info = video_url['url']
if "manifest.f4m" in stream_info:
continue
elif "master.m3u8" in stream_info:
for s, b in M3u8.get_streams_from_m3u8(stream_info, self.proxy, headers=part.HttpHeaders):
item.complete = True
part.append_media_stream(s, b)
# subtitles
subtitles = video_data.get("subtitleReferences")
if subtitles and subtitles[0]["url"]:
Logger.trace(subtitles)
sub_url = subtitles[0]["url"]
file_name = "%s.srt" % (EncodingHelper.encode_md5(sub_url),)
sub_data = UriHandler.open(sub_url, proxy=self.proxy)
# correct the subs
regex = re.compile(r"^1(\d:)", re.MULTILINE)
sub_data = re.sub(regex, r"0\g<1>", sub_data)
sub_data = re.sub(r"--> 1(\d):", r"--> 0\g<1>:", sub_data)
local_complete_path = os.path.join(Config.cacheDir, file_name)
Logger.debug("Saving subtitle to: %s", local_complete_path)
with open(local_complete_path, 'w') as f:
f.write(sub_data)
part.Subtitle = local_complete_path
item.complete = True
return item