def LoadChannelData(self, data):
""" Adds the channel items to the listing.
@param data: The data to use.
Returns a list of MediaItems that were retrieved.
"""
items = []
# data = UriHandler.Open("https://www.svtplay.se/api/channel_page", proxy=self.proxy, noCache=True)
now = datetime.datetime.now()
try:
serverTime = UriHandler.Open("https://www.svtplay.se/api/server_time",
proxy=self.proxy, noCache=True)
serverTimeJson = JsonHelper(serverTime)
serverTime = serverTimeJson.GetValue("time")
except:
Logger.Error("Error determining server time", exc_info=True)
serverTime = "%04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
data = UriHandler.Open(
"https://www.svtplay.se/api/channel_page?now=%s" % (serverTime, ),
proxy=self.proxy)
return data, items
def update_json_video(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item: %s', item.name)
data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=self.httpHeaders)
json_data = JsonHelper(data)
streams = json_data.get_value("formats")
if not streams:
return item
qualities = {"720p": 1600, "480p": 1200, "360p": 500, "other": 0} # , "http-hls": 1500, "3gp-mob01": 300, "flv-web01": 500}
part = item.create_new_empty_media_part()
urls = []
for stream in streams:
url = stream["url"].values()[-1]
if url in urls:
# duplicate url, ignore
continue
urls.append(url)
# actually process the url
if not url.endswith(".m3u8"):
part.append_media_stream(
url=url,
bitrate=qualities.get(stream.get("name", "other"), 0)
)
item.complete = True
# elif AddonSettings.use_adaptive_stream_add_on():
# content_type, url = UriHandler.header(url, self.proxy)
# stream = part.append_media_stream(url, 0)
# M3u8.SetInputStreamAddonInput(stream, self.proxy)
# item.complete = True
else:
content_type, url = UriHandler.header(url, self.proxy)
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def update_video_item(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv
# http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url, proxy=self.proxy)
part = item.create_new_empty_media_part()
if "mediazone.vrt.be" not in item.url:
# Extract actual media data
video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0]
url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, )
data = UriHandler.open(url, proxy=self.proxy)
json = JsonHelper(data)
urls = json.get_value("targetUrls")
for url_info in urls:
Logger.trace(url_info)
if url_info["type"].lower() != "hls":
continue
hls_url = url_info["url"]
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
return item
def load_channel_data(self, data):
""" Adds the channel items to the listing.
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
now = datetime.datetime.now()
try:
server_time = UriHandler.open("https://www.svtplay.se/api/server_time",
proxy=self.proxy, no_cache=True)
server_time_json = JsonHelper(server_time)
server_time = server_time_json.get_value("time")
except:
Logger.error("Error determining server time", exc_info=True)
server_time = "%04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
data = UriHandler.open(
"https://www.svtplay.se/api/channel_page?now=%s" % (server_time, ),
proxy=self.proxy)
return data, items
def UpdateVideoItem(self, item):
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName)
# we need to fetch the actual url as it might differ for single video items
data, secureUrl = UriHandler.Header(item.url, proxy=self.proxy)
secureUrl = secureUrl.rstrip("/")
secureUrl = "%s.mssecurevideo.json" % (secureUrl, )
data = UriHandler.Open(secureUrl, proxy=self.proxy, additionalHeaders=item.HttpHeaders)
secureData = JsonHelper(data, logger=Logger.Instance())
mzid = secureData.GetValue(secureData.json.keys()[0], "videoid")
assetUrl = "https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s" % (mzid, )
data = UriHandler.Open(assetUrl, proxy=self.proxy)
assetData = JsonHelper(data, logger=Logger.Instance())
for streamData in assetData.GetValue("targetUrls"):
if streamData["type"] != "HLS":
continue
part = item.CreateNewEmptyMediaPart()
for s, b, a in M3u8.GetStreamsFromM3u8(streamData["url"], self.proxy, mapAudio=True):
item.complete = True
if a:
audioPart = a.rsplit("-", 1)[-1]
audioPart = "-%s" % (audioPart, )
s = s.replace(".m3u8", audioPart)
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
return item
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
videoData = UriHandler.Open(item.url, proxy=self.proxy)
if not videoData:
return item
videoData = JsonHelper(videoData)
videoInfo = videoData.GetValue("data", "attributes")
part = item.CreateNewEmptyMediaPart()
# Somehow only this specific user-agent works (dunno why)!
part.HttpHeaders[
"user-agent"] = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)"
m3u8url = videoInfo["streaming"]["hls"]["url"]
m3u8data = UriHandler.Open(m3u8url, self.proxy)
for s, b, a in M3u8.GetStreamsFromM3u8(m3u8url,
self.proxy,
appendQueryString=False,
mapAudio=True,
playListData=m3u8data):
item.complete = True
if a:
audioPart = a.split("-prog_index.m3u8", 1)[0]
audioId = audioPart.rsplit("/", 1)[-1]
s = s.replace("-prog_index.m3u8",
"-{0}-prog_index.m3u8".format(audioId))
part.AppendMediaStream(s, b)
vttUrl = M3u8.GetSubtitle(m3u8url, self.proxy, m3u8data)
# https://dplaynordics-vod-80.akamaized.net/dplaydni/259/0/hls/243241001/1112635959-prog_index.m3u8?version_hash=bb753129&hdnts=st=1518218118~exp=1518304518~acl=/*~hmac=bdeefe0ec880f8614e14af4d4a5ca4d3260bf2eaa8559e1eb8ba788645f2087a
vttUrl = vttUrl.replace("-prog_index.m3u8", "-0.vtt")
part.Subtitle = SubtitleHelper.DownloadSubtitle(vttUrl,
format='srt',
proxy=self.proxy)
return item
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName)
data = UriHandler.Open(item.url, proxy=self.proxy)
# get the playlist GUID
playlistGuids = Regexer.DoRegex("<div[^>]+data-playlist-id='([^']+)'[^>]+></div>", data)
if not playlistGuids:
# let's try the alternative then (for the new channels)
playlistGuids = Regexer.DoRegex('local_playlist[", -]+([a-f0-9]{20})"', data)
playlistGuid = playlistGuids[0]
# Logger.Trace(playlistGuid)
# now we can get the playlist meta data
# http://api.mtvnn.com/v2/mrss.xml?uri=mgid%3Asensei%3Avideo%3Amtvnn.com%3Alocal_playlist-39ce0652b0b3c09258d9-SE-uma_site--ad_site-nickelodeon.se-ad_site_referer-video/9764-barjakt&adSite=nickelodeon.se&umaSite={umaSite}&show_images=true&url=http%3A//www.nickelodeon.se/video/9764-barjakt
# but this seems to work.
# http://api.mtvnn.com/v2/mrss.xml?uri=mgid%3Asensei%3Avideo%3Amtvnn.com%3Alocal_playlist-39ce0652b0b3c09258d9
playListUrl = "http://api.mtvnn.com/v2/mrss.xml?uri=mgid%3Asensei%3Avideo%3Amtvnn.com%3Alocal_playlist-" + playlistGuid
playListData = UriHandler.Open(playListUrl, proxy=self.proxy)
# now get the real RTMP data
rtmpMetaData = Regexer.DoRegex("<media:content [^>]+url='([^']+)'", playListData)[0]
rtmpData = UriHandler.Open(rtmpMetaData, proxy=self.proxy)
rtmpUrls = Regexer.DoRegex('<rendition[^>]+bitrate="(\d+)"[^>]*>\W+<src>([^<]+ondemand)/([^<]+)</src>', rtmpData)
part = item.CreateNewEmptyMediaPart()
for rtmpUrl in rtmpUrls:
url = "%s/%s" % (rtmpUrl[1], rtmpUrl[2])
bitrate = rtmpUrl[0]
# convertedUrl = url.replace("ondemand/","ondemand?slist=")
convertedUrl = self.GetVerifiableVideoUrl(url)
part.AppendMediaStream(convertedUrl, bitrate)
item.complete = True
Logger.Trace("Media url: %s", item)
return item
def __ignore_cookie_law(self):
""" Accepts the cookies from UZG in order to have the site available """
Logger.info("Setting the Cookie-Consent cookie for www.uitzendinggemist.nl")
# a second cookie seems to be required
UriHandler.set_cookie(name='npo_cc', value='tmp', domain='.nos.nl')
return
def __ignore_cookie_law(self):
""" Accepts the cookies from UZG in order to have the site available """
Logger.info("Setting the Cookie-Consent cookie for www.dumpert.nl")
# Set-Cookie: cpc=10; path=/; domain=www.dumpert.nl; expires=Thu, 11-Jun-2020 18:49:38 GMT
UriHandler.set_cookie(name='cpc', value='10', domain='.www.dumpert.nl')
return
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
metaData = UriHandler.Open(item.url,
proxy=self.proxy,
referer=self.baseUrl)
meta = JsonHelper(metaData)
streamParts = meta.GetValue("feed", "items")
for streamPart in streamParts:
streamUrl = streamPart["group"]["content"]
# streamUrl = streamUrl.replace("{device}", "ipad")
streamUrl = streamUrl.replace("{device}", "html5")
streamUrl = "%s&format=json" % (streamUrl, )
streamData = UriHandler.Open(streamUrl, proxy=self.proxy)
stream = JsonHelper(streamData)
# subUrls = stream.GetValue("package", "video", "item", 0, "transcript", 0, "typographic")
part = item.CreateNewEmptyMediaPart()
# m3u8Url = stream.GetValue("package", "video", "item", 0, "rendition", 0, "src")
# for s, b in M3u8.GetStreamsFromM3u8(m3u8Url, self.proxy):
# item.complete = True
# part.AppendMediaStream(s, b)
rtmpDatas = stream.GetValue("package", "video", "item", 0,
"rendition")
for rtmpData in rtmpDatas:
rtmpUrl = rtmpData["src"]
rtmpUrl = rtmpUrl.replace("rtmpe://", "rtmp://")
bitrate = rtmpData["bitrate"]
part.AppendMediaStream(rtmpUrl, bitrate)
item.complete = True
Logger.Trace("Media url: %s", item)
return item
def send_files(self, name, file_paths):
""" Sends multiple files.
:param str|unicode name: Name for the gist/paste.
:param list[str|unicode] file_paths: List of file paths.
:return: The result of the upload.
:rtype: any
"""
if self.__mode != "gist":
raise ValueError("Invalid mode for multiple files")
params = {
"description": name,
"public": False,
"files": {
# name: {
# "content": code
# }
}
}
for file_path in file_paths:
if not os.path.isfile(file_path):
continue
code = self.__read_file_bytes(file_path)
file_name = os.path.split(file_path)
params["files"][file_name[-1]] = {"content": code}
headers = {
"Content-Type": "application/json"
}
post_data = JsonHelper.dump(params, pretty_print=False)
data = UriHandler.open("https://api.github.com/gists", params=post_data,
proxy=self.__proxy, additional_headers=headers)
if not data:
raise IOError("Error posting Gist to GitHub")
json_data = JsonHelper(data)
url = json_data.get_value("html_url")
if self.__logger:
self.__logger.info("Gist: %s", url)
# minify with google
# POST https://www.googleapis.com/urlshortener/v1/url
# Content-Type: application/json
shortener = {"longUrl": url}
google = "https://www.googleapis.com/urlshortener/v1/url?key=%s" % (self.__apiKey,)
google_data = UriHandler.open(google, params=JsonHelper.dump(shortener, False),
proxy=self.__proxy,
additional_headers={"Content-Type": "application/json"})
google_url = JsonHelper(google_data).get_value("id")
if self.__logger:
self.__logger.info("Goo.gl: %s", google_url)
return google_url
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
if not item.url.endswith(".js"):
data = UriHandler.open(item.url, proxy=self.proxy)
data_id = Regexer.do_regex(r'data-id="(\d+)"[^>]+data-playout', data)
if data_id is None:
Logger.warning("Cannot find stream-id for L1 stream.")
return item
data_url = "https://l1.bbvms.com/p/video/c/{}.json".format(data_id[0])
else:
data_url = item.url
data = UriHandler.open(data_url, proxy=self.proxy)
json = JsonHelper(data, logger=Logger.instance())
Logger.trace(json)
base_url = json.get_value("publicationData", "defaultMediaAssetPath")
streams = json.get_value("clipData", "assets")
item.MediaItemParts = []
part = item.create_new_empty_media_part()
for stream in streams:
url = stream.get("src", None)
if "://" not in url:
url = "{}{}".format(base_url, url)
bitrate = stream.get("bandwidth", None)
if url:
part.append_media_stream(url, bitrate)
if not item.thumb and json.get_value("thumbnails"):
url = json.get_value("thumbnails")[0].get("src", None)
if url and "http:/" not in url:
url = "%s%s" % (self.baseUrl, url)
item.thumb = url
item.complete = True
return item
def DownloadVideoItem(self, item):
"""Downloads an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that should be downloaded.
Returns:
The original item with more data added to it's properties.
Used to download an <item>. If the item is not complete, the self.UpdateVideoItem
method is called to update the item. The method downloads only the MediaStream
with the bitrate that was set in the addon settings.
After downloading the self.downloaded property is set.
"""
if not item.IsPlayable():
Logger.Error("Cannot download a folder item.")
return item
if item.IsPlayable():
if not item.complete:
Logger.Info("Fetching MediaUrl for PlayableItem[%s]", item.type)
item = self.ProcessVideoItem(item)
if not item.complete or not item.HasMediaItemParts():
Logger.Error("Cannot download incomplete item or item without MediaItemParts")
return item
i = 1
bitrate = AddonSettings.GetMaxStreamBitrate()
for mediaItemPart in item.MediaItemParts:
Logger.Info("Trying to download %s", mediaItemPart)
stream = mediaItemPart.GetMediaStreamForBitrate(bitrate)
downloadUrl = stream.Url
extension = UriHandler.GetExtensionFromUrl(downloadUrl)
if len(item.MediaItemParts) > 1:
saveFileName = "%s-Part_%s.%s" % (item.name, i, extension)
else:
saveFileName = "%s.%s" % (item.name, extension)
Logger.Debug(saveFileName)
# headers = item.HttpHeaders + mediaItemPart.HttpHeaders
headers = item.HttpHeaders.copy()
headers.update(mediaItemPart.HttpHeaders)
progressDialog = XbmcDialogProgressWrapper("Downloading Item", item.name, stream.Url)
folderName = XbmcWrapper.ShowFolderSelection('Select download destination for "%s"' % (saveFileName, ))
UriHandler.Download(downloadUrl, saveFileName, folderName, progressDialog, proxy=self.proxy,
additionalHeaders=headers)
i += 1
item.downloaded = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
meta_data = UriHandler.open(item.url,
proxy=self.proxy,
referer=self.baseUrl)
meta = JsonHelper(meta_data)
stream_parts = meta.get_value("feed", "items")
for stream_part in stream_parts:
stream_url = stream_part["group"]["content"]
stream_url = stream_url.replace("{device}", "html5")
stream_url = "%s&format=json" % (stream_url, )
stream_data = UriHandler.open(stream_url, proxy=self.proxy)
stream = JsonHelper(stream_data)
# subUrls = stream.get_value("package", "video", "item", 0, "transcript", 0, "typographic") # NOSONAR
part = item.create_new_empty_media_part()
# m3u8Url = stream.get_value("package", "video", "item", 0, "rendition", 0, "src") # NOSONAR
# for s, b in M3u8.get_streams_from_m3u8(m3u8Url, self.proxy):
# item.complete = True
# part.append_media_stream(s, b)
rtmp_datas = stream.get_value("package", "video", "item", 0,
"rendition")
for rtmp_data in rtmp_datas:
rtmp_url = rtmp_data["src"]
rtmp_url = rtmp_url.replace("rtmpe://", "rtmp://")
bitrate = rtmp_data["bitrate"]
part.append_media_stream(rtmp_url, bitrate)
item.complete = True
Logger.trace("Media url: %s", item)
return item
def UpdateVideoItem(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName)
# get additional info
data = UriHandler.Open(item.url, proxy=self.proxy)
guid = Regexer.DoRegex('<meta property="og:video" content="http://player.extreme.com/FCPlayer.swf\?id=([^&]+)&[^"]+" />', data)
#<param name="flashvars" value="id=dj0xMDEzNzQyJmM9MTAwMDAwNA&tags=source%253Dfreecaster&autoplay=1" />
# http://freecaster.tv/player/smil/dj0xMDEzNzQyJmM9MTAwMDAwNA -> playlist with bitrate
# http://freecaster.tv/player/smil/dj0xMDEzNzQyJmM9MTAwMDAwNA -> info (not needed, get description from main page.
if len(guid) > 0:
url = '%s/player/smil/%s' % (self.baseUrl, guid[0],)
data = UriHandler.Open(url)
smiller = Smil(data)
baseUrl = smiller.GetBaseUrl()
urls = smiller.GetVideosAndBitrates()
part = item.CreateNewEmptyMediaPart()
for url in urls:
if "youtube" in url[0]:
for s, b in YouTube.GetStreamsFromYouTube(url[0], self.proxy):
item.complete = True
part.AppendMediaStream(s, b)
else:
part.AppendMediaStream("%s%s" % (baseUrl, url[0]), bitrate=int(int(url[1]) / 1000))
item.complete = True
Logger.Trace("UpdateVideoItem complete: %s", item)
return item
# Try the brightcove
brightCoveRegex = '<object id="myExperience[\w\W]+?videoPlayer" value="(\d+)"[\w\W]{0,1000}?playerKey" value="([^"]+)'
brightCoveData = Regexer.DoRegex(brightCoveRegex, data)
Logger.Trace(brightCoveData)
if len(brightCoveData) > 0:
seed = "c5f9ae8729f7054d43187989ef3421531ee8678d"
objectData = brightCoveData[0]
# from proxyinfo import ProxyInfo
playerKey = str(objectData[1])
videoId = int(objectData[0])
part = item.CreateNewEmptyMediaPart()
# But we need the IOS streams!
amfHelper = BrightCove(Logger.Instance(), playerKey, videoId, str(item.url), seed, proxy=self.proxy)
for stream, bitrate in amfHelper.GetStreamInfo(renditions="IOSRenditions"):
part.AppendMediaStream(stream, bitrate)
# Logger.Error("Cannot find GUID in url: %s", item.url)
return item
def __ignore_cookie_law(self):
""" Accepts the cookies from RTL channel in order to have the site available """
Logger.info(
"Setting the Cookie-Consent cookie for www.uitzendinggemist.nl")
# the rfc2109 parameters is not valid in Python 2.4 (Xbox), so we ommit it.
UriHandler.set_cookie(name='rtlcookieconsent',
value='yes',
domain='.www.rtl.nl')
return
def UpdateJsonVideoItem(self, item):
data = UriHandler.Open(item.url, proxy=self.proxy)
json = JsonHelper(data)
m3u8Url = json.GetValue("playlist")
if m3u8Url != "https://embed.kijk.nl/api/playlist/.m3u8":
part = item.CreateNewEmptyMediaPart()
for s, b in M3u8.GetStreamsFromM3u8(m3u8Url,
self.proxy,
appendQueryString=True):
if "_enc_" in s:
Logger.Warning("Found encrypted stream. Skipping %s", s)
continue
item.complete = True
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
return item
Logger.Warning("No M3u8 data found. Falling back to BrightCove")
videoId = json.GetValue("vpakey")
# videoId = json.GetValue("videoId") -> Not all items have a videoId
url = "https://embed.kijk.nl/video/%s?width=868&height=491" % (
videoId, )
referer = "https://embed.kijk.nl/video/%s" % (videoId, )
part = item.CreateNewEmptyMediaPart()
# First try the new BrightCove JSON
data = UriHandler.Open(url, proxy=self.proxy, referer=referer)
brightCoveRegex = '<video[^>]+data-video-id="(?<videoId>[^"]+)[^>]+data-account="(?<videoAccount>[^"]+)'
brightCoveData = Regexer.DoRegex(Regexer.FromExpresso(brightCoveRegex),
data)
if brightCoveData:
Logger.Info("Found new BrightCove JSON data")
brightCoveUrl = 'https://edge.api.brightcove.com/playback/v1/accounts/%(videoAccount)s/videos/%(videoId)s' % \
brightCoveData[0]
headers = {
"Accept":
"application/json;pk=BCpkADawqM3ve1c3k3HcmzaxBvD8lXCl89K7XEHiKutxZArg2c5RhwJHJANOwPwS_4o7UsC4RhIzXG8Y69mrwKCPlRkIxNgPQVY9qG78SJ1TJop4JoDDcgdsNrg"
}
brightCoveData = UriHandler.Open(brightCoveUrl,
proxy=self.proxy,
additionalHeaders=headers)
brightCoveJson = JsonHelper(brightCoveData)
streams = filter(lambda d: d["container"] == "M2TS",
brightCoveJson.GetValue("sources"))
if streams:
# noinspection PyTypeChecker
streamUrl = streams[0]["src"]
for s, b in M3u8.GetStreamsFromM3u8(streamUrl, self.proxy):
item.complete = True
part.AppendMediaStream(s, b)
return item
def __IgnoreCookieLaw(self):
""" Accepts the cookies from UZG in order to have the site available """
Logger.Info("Setting the Cookie-Consent cookie for www.uitzendinggemist.nl")
UriHandler.SetCookie(name='site_cookie_consent', value='yes',
domain='.www.uitzendinggemist.nl')
UriHandler.SetCookie(name='npo_cc', value='tmp', domain='.www.uitzendinggemist.nl')
UriHandler.SetCookie(name='site_cookie_consent', value='yes', domain='.npo.nl')
UriHandler.SetCookie(name='npo_cc', value='30', domain='.npo.nl')
return
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
url = item.url
data = UriHandler.Open(url)
if "json" in self.mainListUri:
metaData = data
else:
mgid = Regexer.DoRegex("mgid:[^ ]+playlist-[abcdef0-9]+", data)[0]
mgidUrlEncoded = htmlentityhelper.HtmlEntityHelper.UrlEncode(mgid)
metaData = UriHandler.Open(
"http://api.mtvnn.com/v2/mrss.xml?uri=%s" % (mgidUrlEncoded, ))
videoUrl = Regexer.DoRegex("<media:content[^>]+url='([^']+)'>",
metaData)[0]
Logger.Trace(videoUrl)
videoData = UriHandler.Open(videoUrl)
videoItems = Regexer.DoRegex(
'<rendition[^>]+bitrate="(\d+)"[^>]*>\W+<src>([^<]+)<', videoData)
item.MediaItemParts = []
part = item.CreateNewEmptyMediaPart()
for videoItem in videoItems:
mediaUrl = self.GetVerifiableVideoUrl(videoItem[1])
part.AppendMediaStream(mediaUrl, videoItem[0])
item.complete = True
return item
def authenticate(self, username, password):
# Step 1: First initiate an authentication request
auth_request = self.__get_authentication_request(username)
auth_data = JsonHelper.dump(auth_request)
auth_headers = {
"X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
"Accept-Encoding": "identity",
"Content-Type": "application/x-amz-json-1.1"
}
auth_response = UriHandler.open(self.url,
proxy=self.__proxy,
params=auth_data,
additional_headers=auth_headers)
auth_response_json = JsonHelper(auth_response)
challenge_parameters = auth_response_json.get_value(
"ChallengeParameters")
if self.__logger:
self.__logger.trace(challenge_parameters)
challenge_name = auth_response_json.get_value("ChallengeName")
if not challenge_name == "PASSWORD_VERIFIER":
if self.__logger:
self.__logger.error("Cannot start authentication challenge")
return None
# Step 2: Respond to the Challenge with a valid ChallengeResponse
challenge_request = self.__get_challenge_response_request(
challenge_parameters, password)
challenge_data = JsonHelper.dump(challenge_request)
challenge_headers = {
"X-Amz-Target":
"AWSCognitoIdentityProviderService.RespondToAuthChallenge",
"Content-Type": "application/x-amz-json-1.1"
}
auth_response = UriHandler.open(self.url,
proxy=self.__proxy,
params=challenge_data,
additional_headers=challenge_headers)
auth_response_json = JsonHelper(auth_response)
if "message" in auth_response_json.json:
self.__logger.error("Error logging in: %s",
auth_response_json.get_value("message"))
return None, None
id_token = auth_response_json.get_value("AuthenticationResult",
"IdToken")
refresh_token = auth_response_json.get_value("AuthenticationResult",
"RefreshToken")
return id_token, refresh_token
def update_music_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_music_item for %s (%s)', item.name, self.channelName)
url, data = item.url.split("?")
data = UriHandler.open(url, proxy=self.proxy, params=data, additional_headers=item.HttpHeaders)
Logger.trace(data)
json_data = JsonHelper(data)
url = json_data.get_value("url", fallback=None)
if url:
item.append_single_stream(url)
item.Complete = True
return item
def add_episode_paging(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
# we need to create page items. So let's just spoof the paging. Youtube has
# a 50 max results per query limit.
items_per_page = 50
data = UriHandler.open(self.mainListUri, proxy=self.proxy)
xml = xmlhelper.XmlHelper(data)
nr_items = xml.get_single_node_content("openSearch:totalResults")
for index in range(1, int(nr_items), items_per_page):
items.append(self.create_episode_item([index, items_per_page]))
# Continue working normal!
return data, items
def UpdateVideoItem(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
if not item.url.endswith("m3u8"):
data = UriHandler.Open(item.url, proxy=self.proxy)
jsonData = Regexer.DoRegex(self.mediaUrlRegex, data)
if not jsonData:
Logger.Error("Cannot find JSON stream info.")
return item
json = JsonHelper(jsonData[0])
Logger.Trace(json.json)
stream = json.GetValue("source", "hls")
Logger.Debug("Found HLS: %s", stream)
else:
stream = item.url
part = item.CreateNewEmptyMediaPart()
for s, b in M3u8.GetStreamsFromM3u8(stream, self.proxy):
item.complete = True
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
# var playerConfig = {"id":"mediaplayer","width":"100%","height":"100%","autostart":"false","image":"http:\/\/www.ketnet.be\/sites\/default\/files\/thumb_5667ea22632bc.jpg","brand":"ketnet","source":{"hls":"http:\/\/vod.stream.vrt.be\/ketnet\/_definst_\/mp4:ketnet\/2015\/12\/Ben_ik_familie_van_R001_A0023_20151208_143112_864.mp4\/playlist.m3u8"},"analytics":{"type_stream":"vod","playlist":"Ben ik familie van?","program":"Ben ik familie van?","episode":"Ben ik familie van?: Warre - Aflevering 3","parts":"1","whatson":"270157835527"},"title":"Ben ik familie van?: Warre - Aflevering 3","description":"Ben ik familie van?: Warre - Aflevering 3"}
return item
def AddEpisodePaging(self, data):
"""Performs pre-process actions for data processing/
Arguments:
data : string - the retrieve data that was loaded for the current item and URL.
Returns:
A tuple of the data and a list of MediaItems that were generated.
Accepts an data from the ProcessFolderList method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
"""
items = []
# we need to create page items. So let's just spoof the paging. Youtube has
# a 50 max results per query limit.
itemsPerPage = 50
data = UriHandler.Open(self.mainListUri, proxy=self.proxy)
xml = xmlhelper.XmlHelper(data)
nrItems = xml.GetSingleNodeContent("openSearch:totalResults")
for index in range(1, int(nrItems), itemsPerPage):
items.append(self.CreateEpisodeItem([index, itemsPerPage]))
pass
# Continue working normal!
return data, items
def UpdateLiveItem(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.Open(item.url, proxy=self.proxy)
data = data[data.index('{'):]
json = JsonHelper(data)
sid = json.GetValue("sid")
videoUrl = "http://edge2.tikilive.com:1935/html5_tikilive/34967/amlst:mainstream/playlist.m3u8" \
"?i=YXBwTmFtZT1QbGF5ZXImY0lEPTM0OTY3JmNOYW1lPUFUViUyME5ldHdvcmtzJm9JRD0xMzY1NTUmb05hbW" \
"U9YXR2bmV0d29ya3Mmc0lkPWZvODRpbDNlN3FzNjh1ZXQycWwyZWF2MDgxJnVJRD0wJnVOYW1lPUd1ZXN0MTNiNjk=&id=%s" \
% (sid,)
part = item.CreateNewEmptyMediaPart()
for s, b in M3u8.GetStreamsFromM3u8(videoUrl, self.proxy):
item.complete = True
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
item.complete = True
return item
def UpdateLiveStream(self, item):
Logger.Debug("Updating Live stream")
# let's request a token
token = self.__GetToken()
# What is the channel name to play
channel = self.channelCode
if self.channelCode == "q2":
channel = "2be"
elif self.channelCode == "stievie":
channel = item.metaData["channelId"]
url = "https://stream-live.medialaan.io/stream-live/v1/channels/%s/broadcasts/current/video/?deviceId=%s" % (
channel,
uuid.uuid4() # Could be a random int
)
auth = {"Authorization": "apikey=%s&access_token=%s" % (self.__apiKey, token)}
data = UriHandler.Open(url, proxy=self.proxy, noCache=True, additionalHeaders=auth)
jsonData = JsonHelper(data)
hls = jsonData.GetValue("response", "url", "hls")
if not hls:
return item
part = item.CreateNewEmptyMediaPart()
for s, b in M3u8.GetStreamsFromM3u8(hls, self.proxy):
item.complete = True
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
return item
def UpdateVideoItem(self, item):
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.Open(item.url,
proxy=self.proxy,
additionalHeaders=item.HttpHeaders)
json = JsonHelper(data)
part = item.CreateNewEmptyMediaPart()
part.Subtitle = NpoStream.GetSubtitle(json.GetValue("mid"),
proxy=self.proxy)
for stream in json.GetValue("videoStreams"):
if not stream["url"].startswith("odi"):
part.AppendMediaStream(stream["url"], stream["bitrate"] / 1000)
item.complete = True
if item.HasMediaItemParts():
return item
for s, b in NpoStream.GetStreamsFromNpo(None,
json.GetValue("mid"),
proxy=self.proxy):
item.complete = True
part.AppendMediaStream(s, b)
return item
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
# https://api.viervijfzes.be/content/c58996a6-9e3d-4195-9ecf-9931194c00bf
# videoId = item.url.split("/")[-1]
# url = "%s/video/v3/embed/%s" % (self.baseUrl, videoId,)
url = item.url
data = UriHandler.Open(url, proxy=self.proxy)
return self.__UpdateVideo(item, data)
def AddClips(self, data):
Logger.Info("Adding Clips Pre-Processing")
items = []
# if the main list was retrieve using json, are the current data is json, just determine
# the clip URL
clipUrl = None
if data.lstrip().startswith("{"):
if self.parentItem.url.endswith("type=program"):
# http://playapi.mtgx.tv/v3/videos?format=6723&order=-airdate&type=program
# http://playapi.mtgx.tv/v3/videos?format=6723&order=-updated&type=clip" % (dataId,)
clipUrl = self.parentItem.url.replace("type=program",
"type=clip")
else:
# now we determine the ID and load the json data
dataId = Regexer.DoRegex('data-format-id="(\d+)"', data)[-1]
Logger.Debug("Found FormatId = %s", dataId)
programUrl = "http://playapi.mtgx.tv/v3/videos?format=%s&order=-airdate&type=program" % (
dataId, )
data = UriHandler.Open(programUrl, proxy=self.proxy)
clipUrl = "http://playapi.mtgx.tv/v3/videos?format=%s&order=-updated&type=clip" % (
dataId, )
if clipUrl is not None:
clipTitle = LanguageHelper.GetLocalizedString(LanguageHelper.Clips)
clipItem = mediaitem.MediaItem("\a.: %s :." % (clipTitle, ),
clipUrl)
clipItem.thumb = self.noImage
items.append(clipItem)
Logger.Debug("Pre-Processing finished")
return data, items
def __IsNewVersionAvailable(self):
""" Verifies that there is a new version available. It compares the addons.xml.md5 with the stored one.
@return: True or False
"""
# first check if there is a "channel" folder
channelPath = os.path.join(Config.rootDir, "channels")
if not os.path.isdir(channelPath):
Logger.Warning(
"No Channels found at '%s', skipping updates for now.",
channelPath)
return False
onlineData = UriHandler.Open(Config.UpdateUrl)
self.__NewMd5 = onlineData.strip()
self.__OldMd5 = AddonSettings.GetCurrentAddonXmlMd5()
updated = self.__OldMd5 != self.__NewMd5
if updated:
Logger.Info("New updates are available ('%s' vs '%s')",
self.__OldMd5, self.__NewMd5)
else:
Logger.Info("No new updates available, MD5 hashes match")
return updated
