def update_live_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
part = item.create_new_empty_media_part()
hls_url = media_info.get_value("streamUrl")
if hls_url is not None and "m3u8" in hls_url:
Logger.debug("Found HLS url for %s: %s",
media_info.json["streamName"], hls_url)
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
else:
Logger.debug("No HLS url found for %s. Fetching RTMP Token.",
media_info.json["streamName"])
# fetch the token:
token_url = "%s/api/media/streaming?streamname=%s" \
% (self.baseUrl, media_info.json["streamName"])
token_data = UriHandler.open(token_url,
proxy=self.proxy,
additional_headers=item.HttpHeaders,
no_cache=True)
token_data = JsonHelper(token_data)
token = token_data.get_value("token")
Logger.debug("Found token '%s' for '%s'", token,
media_info.json["streamName"])
rtmp_url = "rtmp://rtmp.rtbf.be/livecast/%s?%s pageUrl=%s tcUrl=rtmp://rtmp.rtbf.be/livecast" \
% (media_info.json["streamName"], token, self.baseUrl)
rtmp_url = self.get_verifiable_video_url(rtmp_url)
part.append_media_stream(rtmp_url, 0)
item.complete = True
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url)
urls = Regexer.do_regex(
'<a href="([^"]+.(?:wmv|mp4))">(High|Medium|Mid|Low|MP4)', data)
media_part = item.create_new_empty_media_part()
for url in urls:
if url[1].lower() == "high":
bitrate = 2000
elif url[1].lower() == "medium" or url[1].lower() == "mid":
bitrate = 1200
elif url[1].lower() == "low" or url[1].lower() == "mp4":
bitrate = 200
else:
bitrate = 0
media_part.append_media_stream(
HtmlEntityHelper.convert_html_entities(url[0]), bitrate)
item.complete = True
return item
def get_streams_from_f4m(url, headers=None):
""" Parsers standard F4m lists and returns a list of tuples with streams and bitrates that can be used by
other methods
:param str url: The url to download.
:param dict[str,str] headers: Possible HTTP Headers.
Can be used like this:
part = item.create_new_empty_media_part()
for s, b in F4m.get_streams_from_f4m(url):
item.complete = True
# s = self.get_verifiable_video_url(s)
part.append_media_stream(s, b)
"""
streams = []
data = UriHandler.open(url, additional_headers=headers)
Logger.trace(data)
Logger.debug("Processing F4M Streams: %s", url)
needle = '<media href="([^"]+)"[^>]*bitrate="([^"]+)"'
needles = Regexer.do_regex(needle, data)
base_url_logged = False
base_url = url[:url.rindex("/")]
for n in needles:
# see if we need to append a server path
Logger.trace(n)
if "://" not in n[0]:
if not base_url_logged:
Logger.trace("Using base_url %s for F4M", base_url)
base_url_logged = True
stream = "%s/%s" % (base_url, n[0])
else:
if not base_url_logged:
Logger.trace("Full url found in F4M")
base_url_logged = True
stream = n[0]
bitrate = int(n[1])
streams.append((stream, bitrate))
Logger.debug("Found %s substreams in F4M", len(streams))
return streams
def add_clips(self, data):
""" Add an items that lists clips.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Adding Clips Pre-Processing")
items = []
# if the main list was retrieve using json, are the current data is json, just determine
# the clip URL
clip_url = None
if data.lstrip().startswith("{"):
if self.parentItem.url.endswith("type=program"):
# http://playapi.mtgx.tv/v3/videos?format=6723&order=-airdate&type=program
# http://playapi.mtgx.tv/v3/videos?format=6723&order=-updated&type=clip" % (data_id,)
clip_url = self.parentItem.url.replace("type=program",
"type=clip")
else:
# now we determine the ID and load the json data
data_id = Regexer.do_regex(r'data-format-id="(\d+)"', data)[-1]
Logger.debug("Found FormatId = %s", data_id)
program_url = \
"http://playapi.mtgx.tv/v3/videos?format=%s&order=-airdate&type=program" % (data_id,)
data = UriHandler.open(program_url, proxy=self.proxy)
clip_url = \
"http://playapi.mtgx.tv/v3/videos?format=%s&order=-updated&type=clip" % (data_id,)
if clip_url is not None:
clip_title = LanguageHelper.get_localized_string(
LanguageHelper.Clips)
clip_item = MediaItem("\a.: %s :." % (clip_title, ), clip_url)
clip_item.thumb = self.parentItem.thumb
clip_item.fanart = self.parentItem.fanart
items.append(clip_item)
Logger.debug("Pre-Processing finished")
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
url = Regexer.do_regex(self.mediaUrlRegex, data)[-1]
part = MediaItemPart(item.name, url)
item.MediaItemParts.append(part)
Logger.info('finishing update_video_item. MediaItems are %s', item)
if not item.thumb and self.noImage:
# no thumb was set yet and no url
Logger.debug("Setting thumb to %s", item.thumb)
if not item.has_media_item_parts():
item.complete = False
else:
item.complete = True
return item
def __convert_json_subtitle_to_srt(json_subtitle):
"""Converts Json Subtitle format into SRT format:
Arguments:
jsonSubtitle : string - Json Subtitle subtitle format
Returns:
SRT formatted subtitle:
Example:
{"startMillis":80,"endMillis":4170,"text":"Ett Kanal 5:\nAlla gonblick i \"100 jdare!!!\"?","posX":0.5,"posY":0.9,"colorR":220,"colorG":220,"colorB":220}
Returns
1
00:00:20,000 --> 00:00:24,400
text
The format of the timecode is Hours:Minutes:Seconds:Ticks where a "Tick"
is a value of between 0 and 249 and lasts 4 milliseconds.
"""
regex = r'"startMillis":(\d+),"endMillis":(\d+),"text":"(.+?)(?=["] *,)'
subs = Regexer.do_regex(regex, json_subtitle)
# Init some stuff
srt = ""
i = 1
for sub in subs:
try:
start = SubtitleHelper.__convert_to_time(sub[0])
end = SubtitleHelper.__convert_to_time(sub[1])
text = sub[2].replace('\"', '"')
text = JsonHelper.convert_special_chars(text)
text = HtmlEntityHelper.convert_html_entities(text)
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end,
text.strip())
i += 1
except:
Logger.error("Error parsing subtitle: %s", sub, exc_info=True)
return srt
def get_subtitle(self):
""" Retrieves the URL of the included subtitle
:return: The url for the subtitle
:rtype: str
"""
regex = r'<param\W*name="subtitle"[^>]*value="([^"]+)'
urls = Regexer.do_regex(regex, self.data)
for url in urls:
if "http:" in url:
return url
else:
return "%s/%s" % (self.get_base_url().rstrip("/"),
url.lstrip("/"))
return ""
def extract_json(self, data):
""" Extracts JSON data from pages
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
recent = MediaItem("\a .: Recent :.", "https://www.een.be/deze-week")
recent.type = "folder"
recent.complete = True
recent.dontGroup = True
items.append(recent)
data = Regexer.do_regex(r'epgAZ\W+({"data"[\w\W]+?);<', data)[0]
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
if item.url.startswith("http"):
data = UriHandler.open(item.url, proxy=self.proxy)
json_data = Regexer.do_regex(self.mediaUrlRegex, data)
json = JsonHelper(json_data[0])
mzid = json.get_value("mzid")
if not mzid:
item.url = json.get_value("source", "hls")
return self.__update_from_source(item)
else:
mzid = item.url
hls_over_dash = self._get_setting("hls_over_dash", 'false') == 'true'
from resources.lib.streams.vualto import Vualto
v = Vualto(self, "ketnet@prod")
item = v.get_stream_info(item, mzid, hls_over_dash=hls_over_dash)
return item
def get_videos_and_bitrates(self):
""" Retrieves all video's and bitrates in the Smil file.
In this case:
["myStream500K@54552", "500000"]
["myStream900K@54552", "900000"]
["myStream1500K@54552", "1500000"]
:return: A list of all video's and bitrates in the Smil file.
:rtype: list[list[str,str]]
"""
regex = '<video src="([^"]+)"[^>]+system-bitrate="([^"]+)"'
results = Regexer.do_regex(regex, self.data)
if len(results) > 0:
return results
else:
return None
def extract_json_data(self, data):
""" Extracts the JSON data from the HTML page and passes it back to Retrospect.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_data = Regexer.do_regex(r'("staticpages":\s*{[^<]+),\s*"translations":', data)[0]
# We need to put it in a JSON envelop as we are taking JSON from the 'middle'
return_data = "{{{0}}}".format(json_data)
Logger.trace("Found Json:\n%s", return_data)
return JsonHelper(return_data), items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url)
m3u8_url = Regexer.do_regex('data-file="([^"]+)"', data)[0]
part = item.create_new_empty_media_part()
if AddonSettings.use_adaptive_stream_add_on(with_encryption=False):
stream = part.append_media_stream(m3u8_url, 0)
M3u8.set_input_stream_addon_input(stream)
item.complete = True
else:
for s, b, a in M3u8.get_streams_from_m3u8(m3u8_url, map_audio=True):
if a and "-audio" not in s:
video_part = s.rsplit("-", 1)[-1]
video_part = "-%s" % (video_part,)
s = a.replace(".m3u8", video_part)
part.append_media_stream(s, b)
item.complete = True
return item
def extract_json_data(self, data):
""" Extracts the JSON data for video parsing
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_text = Regexer.do_regex(
r'ProgramDescription" data-react-props="([^"]+)', data)[0]
json_text = HtmlEntityHelper.convert_html_entities(json_text)
json_data = JsonHelper(json_text)
Logger.debug("Pre-Processing finished")
return json_data, items
def __update_video(self, item, data):
if not item.url.startswith("https://api.viervijfzes.be/content/"):
regex = 'data-videoid="([^"]+)'
m3u8_url = Regexer.do_regex(regex, data)[-1]
# we either have an URL now or an uuid
else:
m3u8_url = item.url.rsplit("/", 1)[-1]
if ".m3u8" not in m3u8_url:
Logger.info("Not a direct M3u8 file. Need to log in")
url = "https://api.viervijfzes.be/content/%s" % (m3u8_url, )
# We need to log in
if not self.loggedOn:
self.log_on()
# add authorization header
authentication_header = {
"authorization": self.__idToken,
"content-type": "application/json"
}
data = UriHandler.open(url,
proxy=self.proxy,
additional_headers=authentication_header)
json_data = JsonHelper(data)
m3u8_url = json_data.get_value("video", "S")
# Geo Locked?
if "/geo/" in m3u8_url.lower():
# set it for the error statistics
item.isGeoLocked = True
part = item.create_new_empty_media_part()
item.complete = M3u8.update_part_with_m3u8_streams(part,
m3u8_url,
proxy=self.proxy,
channel=self,
encrypted=False)
return item
def __update_video(self, item, data):
regex = 'data-file="([^"]+)'
m3u8_url = Regexer.do_regex(regex, data)[-1]
if ".m3u8" not in m3u8_url:
Logger.info("Not a direct M3u8 file. Need to log in")
url = "https://api.viervijfzes.be/content/%s" % (m3u8_url, )
# We need to log in
if not self.loggedOn:
self.log_on()
# add authorization header
authentication_header = {
"authorization": self.__idToken,
"content-type": "application/json"
}
data = UriHandler.open(url,
proxy=self.proxy,
additional_headers=authentication_header)
json_data = JsonHelper(data)
m3u8_url = json_data.get_value("video", "S")
# Geo Locked?
if "geo" in m3u8_url.lower():
# set it for the error statistics
item.isGeoLocked = True
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(m3u8_url, self.proxy):
if int(b) < 200:
Logger.info("Skipping stream of quality '%s' kbps", b)
continue
item.complete = True
part.append_media_stream(s, b)
item.complete = True
return item
def add_pages(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Adding pages")
# extract the current page from:
# http://www.foxsports.nl/video/filter/fragments/1/alle/tennis/
current_pages = Regexer.do_regex(r'(.+filter/fragments)/(\d+)/(.+)',
self.parentItem.url)
if not current_pages:
return data, []
current_page = current_pages[0]
items = []
url = "%s/%s/%s" % (current_page[0], int(current_page[1]) + 1,
current_page[2])
page_item = MediaItem(
LanguageHelper.get_localized_string(LanguageHelper.MorePages), url)
page_item.fanart = self.parentItem.fanart
page_item.thumb = self.parentItem.thumb
page_item.dontGroup = True
items.append(page_item)
return data, items
def __convert_ttml_to_srt(ttml):
"""Converts sami format into SRT format:
Arguments:
ttml : string - TTML (Timed Text Markup Language) subtitle format
Returns:
SRT formatted subtitle:
Example:
1
00:00:20,000 --> 00:00:24,400
text
"""
pars_regex = r'<p[^>]+begin="([^"]+)\.(\d+)"[^>]+end="([^"]+)\.(\d+)"[^>]*>([\w\W]+?)</p>'
subs = Regexer.do_regex(pars_regex, ttml)
srt = ""
i = 1
for sub in subs:
try:
start = "%s,%03d" % (sub[0], int(sub[1]))
end = "%s,%03d" % (sub[2], int(sub[3]))
text = sub[4].replace("<br />", "\n")
text = HtmlEntityHelper.convert_html_entities(text)
text = text.replace("\r\n", "")
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end,
text.strip())
i += 1
except:
Logger.error("Error parsing subtitle: %s",
sub[1],
exc_info=True)
return srt
def get_mms_from_html(url, proxy=None, index=0):
"""Opens a URL with a MMS playlist and returns the first found stream
in the MMS file. Searches for http://url and returns mms://url.
Arguments:
url : string - the URL to a MMS playlist.
Keyword Arguments:
proxy : Proxy - Proxy info
index : int - The index of the item to retrieve
Returns:
The first found stream in a MMS playlist. If the <url> ends with .mms
it is assumed to already be a single stream. In that case the URL
is returned.
Example:
Ref1=http://url.here/stream1
Ref2=http://url.here/stream2
Will return: mms://url.here.stream1
"""
if url.find(".mms") > 0:
Logger.info("MMS found in url: %s", url)
return url
Logger.debug("Parsing %s to find MMS", url)
data = UriHandler.open(url, proxy=proxy)
urls = Regexer.do_regex(r"[Rr]ef\d=http://([^\r\n]+)", data)
if len(urls) > index:
return "mms://%s" % (urls[index],)
elif len(urls) > 0:
return "mms://%s" % (urls[0],)
else:
return url
def select_video_section(self, data):
""" Performs pre-process actions for data processing
:param str|unicode data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
end_of_section = data.rfind('<div class="ketnet-abc-index">')
if end_of_section > 0:
data = data[:end_of_section]
# find the first main video
json_data = Regexer.do_regex(self.mediaUrlRegex, data)
if not json_data:
Logger.debug("No show data found as JSON")
return data, items
Logger.trace(json_data[0])
json = JsonHelper(json_data[0])
title = json.get_value("title")
url = json.get_value("source", "hls") or ""
item = MediaItem(title, url)
item.type = 'video'
item.description = json.get_value("description", fallback=None)
item.thumb = json.get_value("image", fallback=self.noImage)
item.fanart = self.parentItem.fanart
item.complete = False
items.append(item)
Logger.debug("Pre-Processing finished")
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# Get the MZID
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
json_data = Regexer.do_regex(
r'<script type="application/ld\+json">(.*?)</script>', data)
json_info = JsonHelper(json_data[-1])
video_id = json_info.get_value("video", "@id")
publication_id = json_info.get_value("publication", -1, "@id")
mzid = "{}${}".format(publication_id, video_id)
return self.update_video_for_mzid(item, mzid)
def update_live_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_live_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=self.httpHeaders)
stream_root = Regexer.do_regex(r'<media href="([^"]+\.isml)', data)[0]
Logger.debug("Found Live stream root: %s", stream_root)
part = item.create_new_empty_media_part()
for s, b in F4m.get_streams_from_f4m(item.url, self.proxy):
item.complete = True
s = s.replace(".f4m", ".m3u8")
part.append_media_stream(s, b)
return item
def update_video_item(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv
# http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url, proxy=self.proxy)
part = item.create_new_empty_media_part()
if "mediazone.vrt.be" not in item.url:
# Extract actual media data
video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]',
data)[0]
url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (
video_id, )
data = UriHandler.open(url, proxy=self.proxy)
json = JsonHelper(data)
urls = json.get_value("targetUrls")
for url_info in urls:
Logger.trace(url_info)
if url_info["type"].lower() != "hls":
continue
hls_url = url_info["url"]
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url)
streams = Regexer.do_regex(self.mediaUrlRegex, data)
item.MediaItemParts = []
part = item.create_new_empty_media_part()
for stream in streams:
Logger.trace(stream)
part.append_media_stream(stream[0], stream[1])
item.complete = True
return item
def extract_tv_show_list(self, data):
""" Performs pre-process actions and converts the dictionary to a proper list
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
# Find the build id for the current CMS build
build_id = Regexer.do_regex(r'"buildId"\W*:\W*"([^"]+)"', data)[0]
data = UriHandler.open("https://www.tv4play.se/_next/data/{}/allprograms.json".format(build_id))
json_data = JsonHelper(data)
json_data.json["pageProps"]["initialApolloState"] = list(json_data.json["pageProps"]["initialApolloState"].values())
return json_data, []
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url, proxy=self.proxy)
stream = Regexer.do_regex(r'data-file="([^"]+)+', data)[0]
part = item.create_new_empty_media_part()
if ".mp3" in stream:
item.complete = True
part.append_media_stream(stream, 0)
elif stream.endswith(".mp4"):
item.complete = True
part.append_media_stream(stream, 2500)
elif ".m3u8" in stream:
item.url = stream
return self.update_live_urls(item)
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# get additional info
data = UriHandler.open(item.url, proxy=self.proxy)
#<param name="flashvars" value="id=dj0xMDEzNzQyJmM9MTAwMDAwNA&tags=source%253Dfreecaster&autoplay=1" />
# http://freecaster.tv/player/smil/dj0xMDEzNzQyJmM9MTAwMDAwNA -> playlist with bitrate
# http://freecaster.tv/player/smil/dj0xMDEzNzQyJmM9MTAwMDAwNA -> info (not needed, get description from main page.
you_tube_url = Regexer.do_regex(
'"(https://www.youtube.com/embed/[^\"]+)', data)
if you_tube_url:
Logger.debug("Using Youtube video")
part = item.create_new_empty_media_part()
you_tube_url = you_tube_url[0].replace("embed/", "watch?v=")
for s, b in YouTube.get_streams_from_you_tube(
you_tube_url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
guid = Regexer.do_regex(
r'<meta property="og:video" content="http://player.extreme.com/FCPlayer.swf\?id=([^&]+)&[^"]+" />',
data)
if len(guid) > 0:
url = '%s/player/smil/%s' % (
self.baseUrl,
guid[0],
)
data = UriHandler.open(url)
smiller = Smil(data)
base_url = smiller.get_base_url()
urls = smiller.get_videos_and_bitrates()
part = item.create_new_empty_media_part()
for url in urls:
if "youtube" in url[0]:
for s, b in YouTube.get_streams_from_you_tube(
url[0], self.proxy):
item.complete = True
part.append_media_stream(s, b)
else:
part.append_media_stream("%s%s" % (base_url, url[0]),
bitrate=int(url[1]) // 1000)
item.complete = True
Logger.trace("update_video_item complete: %s", item)
return item
# Try the brightcove
bright_cove_regex = r'<object id="myExperience[\w\W]+?videoPlayer" value="(\d+)"[\w\W]{0,1000}?playerKey" value="([^"]+)'
bright_cove_data = Regexer.do_regex(bright_cove_regex, data)
Logger.trace(bright_cove_data)
if len(bright_cove_data) > 0:
Logger.error(
"BrightCove AMF is no longer supported (no Py3 library)")
return item
def pre_process_folder_list(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str|unicode data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
# Add a klip folder only on the first page and only if it is not already a clip page
if "type=clip" not in self.parentItem.url \
and "&page=1&" in self.parentItem.url \
and "node_nids=" in self.parentItem.url:
# get the category ID
cat_start = self.parentItem.url.rfind("node_nids=")
cat_id = self.parentItem.url[cat_start + 10:]
Logger.debug("Currently doing CatId: '%s'", cat_id)
url = "https://api.tv4play.se/play/video_assets?platform=tablet&per_page=%s&" \
"type=clip&page=1&node_nids=%s&start=0" % (self.maxPageSize, cat_id,)
clips_title = LanguageHelper.get_localized_string(
LanguageHelper.Clips)
clips = MediaItem(clips_title, url)
clips.complete = True
items.append(clips)
# find the max number of items ("total_hits":2724)
total_items = int(Regexer.do_regex(r'total_hits\W+(\d+)', data)[-1])
Logger.debug("Found total of %s items. Only showing %s.", total_items,
self.maxPageSize)
if total_items > self.maxPageSize and "&page=1&" in self.parentItem.url:
# create a group item
more_title = LanguageHelper.get_localized_string(
LanguageHelper.MorePages)
more = MediaItem(more_title, "")
more.complete = True
items.append(more)
# what are the total number of pages?
current_page = 1
# noinspection PyTypeChecker
total_pages = int(math.ceil(1.0 * total_items / self.maxPageSize))
current_url = self.parentItem.url
needle = "&page="
while current_page < total_pages:
# what is the current page
current_page += 1
url = current_url.replace("%s1" % (needle, ),
"%s%s" % (needle, current_page))
Logger.debug("Adding next page: %s\n%s", current_page, url)
page = MediaItem(str(current_page), url)
page.type = "page"
page.complete = True
if total_pages == 2:
items = [page]
break
else:
more.items.append(page)
Logger.debug("Pre-Processing finished")
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# noinspection PyStatementEffect
"""
data-video-id="1613274"
data-video-type="video"
data-video-src="http://media.vrtnieuws.net/2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-title="Het journaal 1 - 25/04/13"
data-video-rtmp-server="rtmp://vrt.flash.streampower.be/vrtnieuws"
data-video-rtmp-path="2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-rtmpt-server="rtmpt://vrt.flash.streampower.be/vrtnieuws"
data-video-rtmpt-path="2013/04/135132051ONL1304255866693.urlFLVLong.flv"
data-video-iphone-server="http://iphone.streampower.be/vrtnieuws_nogeo/_definst_"
data-video-iphone-path="2013/04/135132051ONL1304255866693.urlMP4_H.264.m4v"
data-video-mobile-server="rtsp://mp4.streampower.be/vrt/vrt_mobile/vrtnieuws_nogeo"
data-video-mobile-path="2013/04/135132051ONL1304255866693.url3GP_MPEG4.3gp"
data-video-sitestat-program="het_journaal_1_-_250413_id_1-1613274"
"""
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url, proxy=self.proxy)
data = data.replace("\\/", "/")
urls = Regexer.do_regex(self.mediaUrlRegex, data)
part = item.create_new_empty_media_part()
for url in urls:
Logger.trace(url)
if url[0] == "src":
flv = url[1]
bitrate = 750
else:
flv_server = url[1]
flv_path = url[2]
if url[0] == "rtmp-server":
flv = "%s//%s" % (flv_server, flv_path)
bitrate = 750
elif url[0] == "rtmpt-server":
continue
# Not working for now
#flv = "%s//%s" % (flv_server, flv_path)
#flv = self.get_verifiable_video_url(flv)
#bitrate = 1500
elif url[0] == "iphone-server":
flv = "%s/%s" % (flv_server, flv_path)
if not flv.endswith("playlist.m3u8"):
flv = "%s/playlist.m3u8" % (flv,)
for s, b in M3u8.get_streams_from_m3u8(flv, self.proxy):
item.complete = True
part.append_media_stream(s, b)
# no need to continue adding the streams
continue
elif url[0] == "mobile-server":
flv = "%s/%s" % (flv_server, flv_path)
bitrate = 250
else:
flv = "%s/%s" % (flv_server, flv_path)
bitrate = 0
part.append_media_stream(flv, bitrate)
item.complete = True
return item
def __update_video_from_brightcove(self, item, data,
use_adaptive_with_encryption):
""" Updates an existing MediaItem with more data based on an MPD stream.
:param str data: Stream info retrieved from BrightCove.
:param bool use_adaptive_with_encryption: Do we use the Adaptive InputStream add-on?
:param MediaItem item: The original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
part = item.create_new_empty_media_part()
# Then try the new BrightCove JSON
bright_cove_regex = '<video[^>]+data-video-id="(?<videoId>[^"]+)[^>]+data-account="(?<videoAccount>[^"]+)'
bright_cove_data = Regexer.do_regex(
Regexer.from_expresso(bright_cove_regex), data)
if not bright_cove_data:
Logger.warning("Error updating using BrightCove data: %s", item)
return item
Logger.info("Found new BrightCove JSON data")
bright_cove_url = 'https://edge.api.brightcove.com/playback/v1/accounts/' \
'%(videoAccount)s/videos/%(videoId)s' % bright_cove_data[0]
headers = {
"Accept":
"application/json;pk=BCpkADawqM3ve1c3k3HcmzaxBvD8lXCl89K7XEHiKutxZArg2c5RhwJHJANOwPwS_4o7UsC4RhIzXG8Y69mrwKCPlRkIxNgPQVY9qG78SJ1TJop4JoDDcgdsNrg"
}
bright_cove_data = UriHandler.open(bright_cove_url,
proxy=self.proxy,
additional_headers=headers)
bright_cove_json = JsonHelper(bright_cove_data)
streams = [
d for d in bright_cove_json.get_value("sources")
if d["container"] == "M2TS"
]
# Old filter
# streams = filter(lambda d: d["container"] == "M2TS", bright_cove_json.get_value("sources"))
if not streams:
Logger.warning("Error extracting streams from BrightCove data: %s",
item)
return item
# noinspection PyTypeChecker
stream_url = streams[0]["src"]
# these streams work better with the the InputStreamAddon because it removes the
# "range" http header
if use_adaptive_with_encryption:
Logger.info("Using InputStreamAddon for playback of HLS stream")
strm = part.append_media_stream(stream_url, 0)
M3u8.set_input_stream_addon_input(strm, proxy=self.proxy)
item.complete = True
return item
for s, b in M3u8.get_streams_from_m3u8(stream_url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def __update_embedded_video(self, item):
""" Updates video items that are encrypted. This could be the default for Krypton!
:param MediaItem item: The item to update.
:return: An updated item.
:rtype: MediaItem
"""
data = UriHandler.open(item.url, proxy=self.proxy)
if UriHandler.instance().status.code == 404:
title, message = Regexer.do_regex(
r'<h1>([^<]+)</h1>\W+<p>([^<]+)<', data)[0]
XbmcWrapper.show_dialog(title, message)
return item
start_needle = "var playerConfig ="
start_data = data.index(start_needle) + len(start_needle)
end_data = data.index("var talpaPlayer")
data = data[start_data:end_data].strip().rstrip(";")
json = JsonHelper(data)
has_drm_only = True
adaptive_available = AddonSettings.use_adaptive_stream_add_on(
with_encryption=False, channel=self)
adaptive_available_encrypted = AddonSettings.use_adaptive_stream_add_on(
with_encryption=True, channel=self)
for play_list_entry in json.get_value("playlist"):
part = item.create_new_empty_media_part()
for source in play_list_entry["sources"]:
stream_type = source["type"]
stream_url = source["file"]
stream_drm = source.get("drm")
if not stream_drm:
has_drm_only = False
if stream_type == "m3u8":
Logger.debug("Found non-encrypted M3u8 stream: %s",
stream_url)
M3u8.update_part_with_m3u8_streams(part,
stream_url,
proxy=self.proxy,
channel=self)
item.complete = True
elif stream_type == "dash" and adaptive_available:
Logger.debug("Found non-encrypted Dash stream: %s",
stream_url)
stream = part.append_media_stream(stream_url, 1)
Mpd.set_input_stream_addon_input(stream,
proxy=self.proxy)
item.complete = True
else:
Logger.debug("Unknown stream source: %s", source)
else:
compatible_drm = "widevine"
if compatible_drm not in stream_drm or stream_type != "dash":
Logger.debug("Found encrypted %s stream: %s",
stream_type, stream_url)
continue
Logger.debug("Found Widevine encrypted Dash stream: %s",
stream_url)
license_url = stream_drm[compatible_drm]["url"]
pid = stream_drm[compatible_drm]["releasePid"]
encryption_json = '{"getRawWidevineLicense":' \
'{"releasePid":"%s", "widevineChallenge":"b{SSM}"}' \
'}' % (pid,)
headers = {
"Content-Type": "application/json",
"Origin": "https://embed.kijk.nl",
"Referer": stream_url
}
encryption_key = Mpd.get_license_key(
license_url,
key_type=None,
key_value=encryption_json,
key_headers=headers)
stream = part.append_media_stream(stream_url, 0)
Mpd.set_input_stream_addon_input(
stream, proxy=self.proxy, license_key=encryption_key)
item.complete = True
subs = [
s['file'] for s in play_list_entry.get("tracks", [])
if s.get('kind') == "captions"
]
if subs:
subtitle = SubtitleHelper.download_subtitle(subs[0],
format="webvtt")
part.Subtitle = subtitle
if has_drm_only and not adaptive_available_encrypted:
XbmcWrapper.show_dialog(
LanguageHelper.get_localized_string(LanguageHelper.DrmTitle),
LanguageHelper.get_localized_string(
LanguageHelper.WidevineLeiaRequired))
return item
