def create_episode_item_api(self, result_set):
""" Creates a new MediaItem for an episode.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'folder'.
:rtype: MediaItem|None
"""
if not isinstance(result_set, dict):
json_data = result_set[1].replace(""", "\"")
result_set = JsonHelper(json_data)
result_set = result_set.json
brand = result_set["brand"]
if brand != self.__channel_brand:
return None
title = result_set["title"]
url = "{}{}".format(self.baseUrl, result_set["link"])
item = MediaItem(title, url)
item.description = result_set["description"]
item.isGeoLocked = True
images = result_set["images"]
item.poster = HtmlEntityHelper.convert_html_entities(images.get("poster"))
item.thumb = HtmlEntityHelper.convert_html_entities(images.get("teaser"))
return item
def create_folder_item(self, result_set):
""" Creates a MediaItem of type 'folder' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'folder'.
:rtype: MediaItem|None
"""
if len(result_set) > 3 and result_set[3] != "":
Logger.debug("Sub category folder found.")
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[3]))
name = "\a.: %s :." % (result_set[4], )
item = MediaItem(name, url)
item.complete = True
item.type = "folder"
return item
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[0]))
name = HtmlEntityHelper.convert_html_entities(result_set[1])
helper = HtmlHelper(result_set[2])
description = helper.get_tag_content("div", {'class': 'description'})
item = MediaItem(name, "%s/RSS" % (url, ))
item.type = 'folder'
item.description = description.strip()
date = helper.get_tag_content("div", {'class': 'date'})
if date == "":
date = helper.get_tag_content("span",
{'class': 'lastPublishedDate'})
if not date == "":
date_parts = Regexer.do_regex(r"(\w+) (\d+)[^<]+, (\d+)", date)
if len(date_parts) > 0:
date_parts = date_parts[0]
month_part = date_parts[0].lower()
day_part = date_parts[1]
year_part = date_parts[2]
try:
month = DateHelper.get_month_from_name(month_part, "en")
item.set_date(year_part, month, day_part)
except:
Logger.error("Error matching month: %s",
month_part,
exc_info=True)
item.complete = True
return item
def create_episode_item(self, result_set):
""" Creates a new MediaItem for an episode.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'folder'.
:rtype: MediaItem|None
"""
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[0]))
name = result_set[1]
if name == "Tags":
return None
if name == "Authors":
return None
if name == "Most Viewed":
return None
if name == "Top Rated":
name = "Recent"
url = "http://channel9.msdn.com/Feeds/RSS"
else:
url = "%s?sort=atoz" % (url, )
item = MediaItem(name, url)
item.complete = True
return item
def __add_breadcrumb(self,
handle,
channel,
selected_item,
last_only=False):
""" Updates the Kodi category with a breadcrumb to the current parent item
:param int handle: The Kodi file handle
:param ChannelInfo|Channel channel: The channel to which the item belongs
:param MediaItem selected_item: The item from which to show the breadcrumbs
:param bool last_only: Show only the last item
"""
bread_crumb = None
if selected_item is not None:
bread_crumb = selected_item.name
elif self.channelObject is not None:
bread_crumb = channel.channelName
if not bread_crumb:
return
bread_crumb = HtmlEntityHelper.convert_html_entities(bread_crumb)
xbmcplugin.setPluginCategory(handle=handle, category=bread_crumb)
def create_episode_item(self, result_set):
""" Creates a new MediaItem for an episode.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'folder'.
:rtype: MediaItem|None
"""
Logger.trace(result_set)
genres = result_set[0]
if self.__genre and self.__genre not in genres:
Logger.debug("Item '%s' filtered due to genre: %s", result_set[2], genres)
return None
url = result_set[1]
if "&" in url:
url = HtmlEntityHelper.convert_html_entities(url)
if not url.startswith("http:"):
url = "%s%s" % (self.baseUrl, url)
# get the ajax page for less bandwidth
url = "%s?sida=1&sort=tid_stigande&embed=true" % (url, )
item = MediaItem(result_set[2], url)
item.complete = True
item.isGeoLocked = True
return item
def __convert_ttml_to_srt(ttml):
"""Converts sami format into SRT format:
Arguments:
ttml : string - TTML (Timed Text Markup Language) subtitle format
Returns:
SRT formatted subtitle:
Example:
1
00:00:20,000 --> 00:00:24,400
text
"""
pars_regex = r'<p[^>]+begin="([^"]+)\.(\d+)"[^>]+end="([^"]+)\.(\d+)"[^>]*>([\w\W]+?)</p>'
subs = Regexer.do_regex(pars_regex, ttml)
srt = ""
i = 1
for sub in subs:
try:
start = "%s,%03d" % (sub[0], int(sub[1]))
end = "%s,%03d" % (sub[2], int(sub[3]))
text = sub[4].replace("<br />", "\n")
text = HtmlEntityHelper.convert_html_entities(text)
text = text.replace("\r\n", "")
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end, text.strip())
i += 1
except:
Logger.error("Error parsing subtitle: %s", sub[1], exc_info=True)
return srt
def update_video_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
# sources
part = item.create_new_empty_media_part()
# high, web, mobile, url
media_sources = media_info.json.get("sources", {})
for quality in media_sources:
url = media_sources[quality]
if quality == "high":
bitrate = 2000
elif quality == "web":
bitrate = 800
elif quality == "mobile":
bitrate = 400
else:
bitrate = 0
part.append_media_stream(url, bitrate)
# geoLocRestriction
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
item.complete = True
return item
def get_kodi_item(self):
""" Creates an Kodi ListItem object for this channel
:return: a Kodi ListItem with all required properties set.
:rtype: xbmcgui.ListItem
"""
name = HtmlEntityHelper.convert_html_entities(self.channelName)
description = HtmlEntityHelper.convert_html_entities(
self.channelDescription)
if self.uses_external_addon:
from resources.lib.xbmcwrapper import XbmcWrapper
name = "{} {}".format(
name, XbmcWrapper.get_external_add_on_label(self.addonUrl))
self.icon = self.__get_image_path(self.icon)
item = kodifactory.list_item(name, description)
item.setArt({'thumb': self.icon, 'icon': self.icon})
# http://mirrors.kodi.tv/docs/python-docs/14.x-helix/xbmcgui.html#ListItem-setInfo
item.setInfo(
"video",
{
"Title": name,
# "Count": self.sortOrderPerCountry,
# "TrackNumber": self.sortOrder,
"Genre": LanguageHelper.get_full_language(self.language),
# "Tagline": description,
"Plot": description
})
if self.poster is not None:
self.poster = self.__get_image_path(self.poster)
item.setArt({'poster': self.poster})
if AddonSettings.hide_fanart():
return item
if self.fanart is not None:
self.fanart = self.__get_image_path(self.fanart)
else:
self.fanart = Config.fanart
item.setArt({'fanart': self.fanart})
return item
def get_kodi_item(self):
""" Creates an Kodi ListItem object for this channel
:return: a Kodi ListItem with all required properties set.
:rtype: xbmcgui.ListItem
"""
name = HtmlEntityHelper.convert_html_entities(self.channelName)
description = HtmlEntityHelper.convert_html_entities(
self.channelDescription)
if self.uses_external_addon:
other = LanguageHelper.get_localized_string(
LanguageHelper.OtherAddon)
name = "{0} {1} [COLOR gold]{2}[/COLOR]".format(
name, unichr(187), other)
self.icon = self.__get_image_path(self.icon)
item = xbmcgui.ListItem(name, description)
item.setArt({'thumb': self.icon, 'icon': self.icon})
# http://mirrors.kodi.tv/docs/python-docs/14.x-helix/xbmcgui.html#ListItem-setInfo
item.setInfo(
"video",
{
"Title": name,
# "Count": self.sortOrderPerCountry,
# "TrackNumber": self.sortOrder,
"Genre": LanguageHelper.get_full_language(self.language),
# "Tagline": description,
"Plot": description
})
if AddonSettings.hide_fanart():
return item
if self.fanart is not None:
self.fanart = self.__get_image_path(self.fanart)
else:
self.fanart = os.path.join(Config.rootDir, "fanart.jpg")
item.setArt({'fanart': self.fanart})
return item
def select_channels(self):
""" Selects the channels that should be visible.
@return: None
"""
valid_channels = ChannelIndex.get_register().get_channels(
include_disabled=True)
channels_to_show = [c for c in valid_channels if c.visible]
# The old way
# channels_to_show = filter(lambda c: c.visible, valid_channels)
selected_channels = [c for c in channels_to_show if c.enabled]
selected_indices = list(
[channels_to_show.index(c) for c in selected_channels])
Logger.debug("Currently selected channels: %s", selected_indices)
channel_to_show_names = [
HtmlEntityHelper.convert_html_entities(c.channelName)
for c in channels_to_show
]
# The old way
# channel_to_show_names = list(map(lambda c: HtmlEntityHelper.convert_html_entities(c.channelName), channels_to_show))
dialog = xbmcgui.Dialog()
heading = LanguageHelper.get_localized_string(
LanguageHelper.ChannelSelection)[:-1]
selected_channels = dialog.multiselect(heading,
channel_to_show_names,
preselect=selected_indices)
if selected_channels is None:
return
selected_channels = list(selected_channels)
Logger.debug("New selected channels: %s", selected_channels)
indices_to_remove = [
i for i in selected_indices if i not in selected_channels
]
indices_to_add = [
i for i in selected_channels if i not in selected_indices
]
for i in indices_to_remove:
Logger.info("Hiding channel: %s", channels_to_show[i])
AddonSettings.set_channel_visiblity(channels_to_show[i], False)
for i in indices_to_add:
Logger.info("Showing channel: %s", channels_to_show[i])
AddonSettings.set_channel_visiblity(channels_to_show[i], True)
self.refresh()
return
def update_live_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
part = item.create_new_empty_media_part()
hls_url = media_info.get_value("streamUrl")
if hls_url is not None and "m3u8" in hls_url:
Logger.debug("Found HLS url for %s: %s",
media_info.json["streamName"], hls_url)
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
else:
Logger.debug("No HLS url found for %s. Fetching RTMP Token.",
media_info.json["streamName"])
# fetch the token:
token_url = "%s/api/media/streaming?streamname=%s" \
% (self.baseUrl, media_info.json["streamName"])
token_data = UriHandler.open(token_url,
proxy=self.proxy,
additional_headers=item.HttpHeaders,
no_cache=True)
token_data = JsonHelper(token_data)
token = token_data.get_value("token")
Logger.debug("Found token '%s' for '%s'", token,
media_info.json["streamName"])
rtmp_url = "rtmp://rtmp.rtbf.be/livecast/%s?%s pageUrl=%s tcUrl=rtmp://rtmp.rtbf.be/livecast" \
% (media_info.json["streamName"], token, self.baseUrl)
rtmp_url = self.get_verifiable_video_url(rtmp_url)
part.append_media_stream(rtmp_url, 0)
item.complete = True
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url)
urls = Regexer.do_regex(
'<a href="([^"]+.(?:wmv|mp4))">(High|Medium|Mid|Low|MP4)', data)
media_part = item.create_new_empty_media_part()
for url in urls:
if url[1].lower() == "high":
bitrate = 2000
elif url[1].lower() == "medium" or url[1].lower() == "mid":
bitrate = 1200
elif url[1].lower() == "low" or url[1].lower() == "mp4":
bitrate = 200
else:
bitrate = 0
media_part.append_media_stream(
HtmlEntityHelper.convert_html_entities(url[0]), bitrate)
item.complete = True
return item
def __convert_json_subtitle_to_srt(json_subtitle):
"""Converts Json Subtitle format into SRT format:
Arguments:
jsonSubtitle : string - Json Subtitle subtitle format
Returns:
SRT formatted subtitle:
Example:
{"startMillis":80,"endMillis":4170,"text":"Ett Kanal 5:\nAlla gonblick i \"100 jdare!!!\"?","posX":0.5,"posY":0.9,"colorR":220,"colorG":220,"colorB":220}
Returns
1
00:00:20,000 --> 00:00:24,400
text
The format of the timecode is Hours:Minutes:Seconds:Ticks where a "Tick"
is a value of between 0 and 249 and lasts 4 milliseconds.
"""
regex = r'"startMillis":(\d+),"endMillis":(\d+),"text":"(.+?)(?=["] *,)'
subs = Regexer.do_regex(regex, json_subtitle)
# Init some stuff
srt = ""
i = 1
for sub in subs:
try:
start = SubtitleHelper.__convert_to_time(sub[0])
end = SubtitleHelper.__convert_to_time(sub[1])
text = sub[2].replace('\"', '"')
text = JsonHelper.convert_special_chars(text)
text = HtmlEntityHelper.convert_html_entities(text)
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end,
text.strip())
i += 1
except:
Logger.error("Error parsing subtitle: %s", sub, exc_info=True)
return srt
def __convert_web_vtt_to_srt(webvvt):
"""Converts sami format into SRT format:
Arguments:
ttml : string - TTML (Timed Text Markup Language) subtitle format
Returns:
SRT formatted subtitle:
Example:
1
00:00:20,000 --> 00:00:24,400
text
"""
count = 0
result = ""
for line in webvvt.split("\n"):
line = line.strip()
if line.endswith("WEBVTT") or line.startswith("X-TIMESTAMP"):
continue
if not line:
continue
if " --> " in line:
count += 1
start, end = line.split(" --> ")
result = "%s\n\n%s" % (result, count)
if start.count(":") == 1:
result = "%s\n00:%s --> 00:%s" % (
result, start.replace(".", ","), end.replace(".", ","))
else:
result = "%s\n%s --> %s" % (result, start.replace(
".", ","), end.replace(".", ","))
elif line == str(count + 1):
# we apparently have built-in numbering using WebVTT cue-numbering
continue
else:
result = "%s\n%s" % (
result, HtmlEntityHelper.convert_html_entities(line))
return result
def extract_json_data(self, data):
""" Extracts the JSON data for video parsing
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_text = Regexer.do_regex(
r'ProgramDescription" data-react-props="([^"]+)', data)[0]
json_text = HtmlEntityHelper.convert_html_entities(json_text)
json_data = JsonHelper(json_text)
Logger.debug("Pre-Processing finished")
return json_data, items
def show_channel_settings(channel):
""" Show the add-on settings and pre-selects the channel settings tab with the correct channel
selected.
:param channel: The channel to display settings for.
"""
channel_name = channel.safe_name
# remove some HTML chars
channel_name = HtmlEntityHelper.convert_html_entities(channel_name)
Logger.debug("Showing channel settings for channel: %s (%s)", channel_name, channel.channelName)
# Set the channel to be the preselected one
AddonSettings.store(KODI).set_setting("config_channel", channel_name)
# show settings and focus on the channel settings tab
if AddonSettings.is_min_version(AddonSettings.KodiLeia):
return AddonSettings.show_settings(-98)
else:
return AddonSettings.show_settings(102)
def __convert_sami_to_srt(sami):
"""Converts sami format into SRT format:
Arguments:
sami : string - SAMI subtitle format
Returns:
SRT formatted subtitle:
Example:
1
00:00:20,000 --> 00:00:24,400
text
"""
pars_regex = r'<sync start="(\d+)"><p[^>]+>([^<]+)</p></sync>\W+<sync start="(\d+)">'
subs = Regexer.do_regex(pars_regex, sami)
if len(subs) == 0:
pars_regex2 = r'<sync start=(\d+)>\W+<p[^>]+>([^\n]+)\W+<sync start=(\d+)>'
subs = Regexer.do_regex(pars_regex2, sami)
srt = ""
i = 1
for sub in subs:
try:
start = SubtitleHelper.__convert_to_time(sub[0])
end = SubtitleHelper.__convert_to_time(sub[2])
text = sub[1]
text = HtmlEntityHelper.convert_html_entities(text)
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end, text)
i += 1
except:
Logger.error("Error parsing subtitle: %s",
sub[1],
exc_info=True)
# re-encode to be able to write it
return srt
def __full_decode_text(self, string_value):
""" Decodes a byte encoded string with HTML content into Unicode String
Arguments:
stringValue : string - The byte encoded string to decode
Returns:
An Unicode String with all HTML entities replaced by their UTF8 characters
The decoding is done by first decode the string to UTF8 and then replace
the HTML entities to their UTF8 characters.
"""
if string_value is None:
return None
if string_value == "":
return ""
# then get rid of the HTML entities
string_value = HtmlEntityHelper.convert_html_entities(string_value)
return string_value
def create_page_item(self, result_set):
""" Creates a MediaItem of type 'page' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'page'.
:rtype: MediaItem|None
"""
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[0]))
item = MediaItem(result_set[self.pageNavigationRegexIndex], url)
item.type = "page"
item.complete = True
Logger.trace("Created '%s' for url %s", item.name, item.url)
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
# noinspection PyStatementEffect
"""
<script type="text/javascript">/* <![CDATA[ */ var movieFlashVars = "
image=http://assets.ur.se/id/147834/images/1_l.jpg
file=/147000-147999/147834-20.mp4
plugins=http://urplay.se/jwplayer/plugins/gapro-1.swf,http://urplay.se/jwplayer/plugins/sharing-2.swf,http://urplay.se/jwplayer/plugins/captions/captions.swf
sharing.link=http://urplay.se/147834
gapro.accountid=UA-12814852-8
captions.margin=40
captions.fontsize=11
captions.back=false
captions.file=http://undertexter.ur.se/147000-147999/147834-19.tt
streamer=rtmp://streaming.ur.se/ondemand
autostart=False"; var htmlVideoElementSource = "http://streaming.ur.se/ondemand/mp4:147834-23.mp4/playlist.m3u8?location=SE"; /* //]]> */ </script>
"""
data = UriHandler.open(item.url)
# Extract stream JSON data from HTML
streams = Regexer.do_regex(
r'ProgramContainer" data-react-props="({[^"]+})"', data)
json_data = streams[0]
json_data = HtmlEntityHelper.convert_html_entities(json_data)
json = JsonHelper(json_data, logger=Logger.instance())
Logger.trace(json.json)
item.MediaItemParts = []
# generic server information
proxy_data = UriHandler.open(
"https://streaming-loadbalancer.ur.se/loadbalancer.json",
no_cache=True)
proxy_json = JsonHelper(proxy_data)
proxy = proxy_json.get_value("redirect")
Logger.trace("Found RTMP Proxy: %s", proxy)
stream_infos = json.get_value("program", "streamingInfo")
part = item.create_new_empty_media_part()
for stream_type, stream_info in stream_infos.items():
Logger.trace(stream_info)
default_stream = stream_info.get("default", False)
bitrates = {
"mp3": 400,
"m4a": 250,
"sd": 1200,
"hd": 2000,
"tt": None
}
for quality, bitrate in bitrates.items():
stream = stream_info.get(quality)
if stream is None:
continue
stream_url = stream["location"]
if quality == "tt":
part.Subtitle = SubtitleHelper.download_subtitle(
stream_url, format="ttml")
continue
bitrate = bitrate if default_stream else bitrate + 1
if stream_type == "raw":
bitrate += 1
url = "https://%s/%smaster.m3u8" % (proxy, stream_url)
part.append_media_stream(url, bitrate)
item.complete = True
return item
def process_folder_list(self, item=None): # NOSONAR
""" Process the selected item and get's it's child items using the available dataparsers.
Accepts an <item> and returns a list of MediaListems with at least name & url
set. The following actions are done:
* determining the correct parsers to use
* call a pre-processor
* parsing the data with the parsers
* calling the creators for item creations
if the item is None, we assume that we are dealing with the first call for this channel and the mainlist uri
is used.
:param MediaItem|None item: The parent item.
:return: A list of MediaItems that form the childeren of the <item>.
:rtype: list[MediaItem]
"""
items = []
self.parentItem = item
if item is None:
Logger.info(
"process_folder_list :: No item was specified. Assuming it was the main channel list"
)
url = self.mainListUri
elif len(item.items) > 0:
return item.items
else:
url = item.url
# Determine the handlers and process
data_parsers = self.__get_data_parsers(url)
# Exclude the updaters only
data_parsers = [
p for p in data_parsers if not p.is_video_updater_only()
]
if [p for p in data_parsers if p.LogOnRequired]:
Logger.info("One or more dataparsers require logging in.")
self.loggedOn = self.log_on()
# now set the headers here and not earlier in case they might have been update by the logon
if item is not None and item.HttpHeaders:
headers = item.HttpHeaders
else:
headers = self.httpHeaders
# Let's retrieve the required data. Main url's
if url.startswith("http:") or url.startswith(
"https:") or url.startswith("file:"):
# Disable cache on live folders
no_cache = item is not None and not item.is_playable(
) and item.isLive
if no_cache:
Logger.debug("Disabling cache for '%s'", item)
data = UriHandler.open(url,
proxy=self.proxy,
additional_headers=headers,
no_cache=no_cache)
# Searching a site using search_site()
elif url == "searchSite" or url == "#searchSite":
Logger.debug("Starting to search")
return self.search_site()
# Labels instead of url's
elif url.startswith("#"):
data = ""
# Others
else:
Logger.debug("Unknown URL format. Setting data to ''")
data = ""
# first check if there is a generic pre-processor
pre_procs = [p for p in data_parsers if p.is_generic_pre_processor()]
num_pre_procs = len(pre_procs)
Logger.trace("Processing %s Generic Pre-Processors DataParsers",
num_pre_procs)
if num_pre_procs > 1:
# warn for strange results if more than 1 generic pre-processor is present.
Logger.warning(
"More than one Generic Pre-Processor is found (%s). They are being processed in the "
"order that Python likes which might result in unexpected result.",
num_pre_procs)
for data_parser in pre_procs:
# remove it from the list
data_parsers.remove(data_parser)
# and process it
Logger.debug("[DataParsers] Pre-Processing %s", data_parser)
(data, pre_items) = data_parser.PreProcessor(data)
items += pre_items
if isinstance(data, JsonHelper):
Logger.debug(
"Generic preprocessor resulted in JsonHelper data")
# Split normal and post-processor data parsers
generic_post_procs = [
p for p in data_parsers if p.is_generic_post_processor()
]
data_parsers = [p for p in data_parsers if p not in generic_post_procs]
# The the other handlers
Logger.trace("Processing %s Normal DataParsers", len(data_parsers))
handler_json = None
for data_parser in data_parsers:
Logger.debug("[DataParsers] Processing %s", data_parser)
# Check for preprocessors
if data_parser.PreProcessor:
Logger.debug(
"[DataParsers] Processing DataParser.PreProcessor")
(handler_data, pre_items) = data_parser.PreProcessor(data)
items += pre_items
else:
handler_data = data
Logger.debug("[DataParsers] Processing DataParser.Parser")
if data_parser.Parser is None or (data_parser.Parser == ""
and not data_parser.IsJson):
if data_parser.Creator:
Logger.warning("No <parser> found for %s. Skipping.",
data_parser.Creator)
continue
if data_parser.IsJson:
if handler_json is None:
# Cache the json requests to improve performance
Logger.trace("Caching JSON results for Dataparsing")
if isinstance(handler_data, JsonHelper):
handler_json = handler_data
else:
handler_json = JsonHelper(handler_data,
Logger.instance())
Logger.trace(data_parser.Parser)
parser_results = handler_json.get_value(fallback=[],
*data_parser.Parser)
if not isinstance(parser_results, (tuple, list)):
# if there is just one match, return that as a list
parser_results = [parser_results]
else:
if isinstance(handler_data, JsonHelper):
raise ValueError(
"Cannot perform Regex Parser on JsonHelper.")
else:
parser_results = Regexer.do_regex(data_parser.Parser,
handler_data)
Logger.debug(
"[DataParsers] Processing DataParser.Creator for %s items",
len(parser_results))
for parser_result in parser_results:
handler_result = data_parser.Creator(parser_result)
if handler_result is not None:
if isinstance(handler_result, list):
items += handler_result
else:
items.append(handler_result)
if data_parser.PostProcessor:
Logger.debug(
"[DataParsers] Processing DataParser.PostProcessor")
if data_parser.IsJson:
items = data_parser.PostProcessor(handler_json, items)
else:
items = data_parser.PostProcessor(handler_data, items)
Logger.trace("Post-processing returned %d items", len(items))
# The post processors
num_post_procs = len(generic_post_procs)
Logger.trace("Processing %s Generic Post-Processors DataParsers",
num_post_procs)
if num_post_procs > 1:
# warn for strange results if more than 1 generic pre-processor is present.
Logger.warning(
"More than one Generic Post-Processor is found (%s). They are being processed in the "
"order that Python likes which might result in unexpected result.",
num_post_procs)
for data_parser in generic_post_procs:
Logger.debug("[DataParsers] Post-processing Generic %s",
data_parser)
items = data_parser.PostProcessor(data, items)
Logger.trace("Post-processing returned %d items", len(items))
# should we exclude DRM/GEO?
hide_geo_locked = AddonSettings.hide_geo_locked_items_for_location(
self.language)
hide_drm_protected = AddonSettings.hide_drm_items()
hide_premium = AddonSettings.hide_premium_items()
hide_folders = AddonSettings.hide_restricted_folders()
type_to_exclude = None
if not hide_folders:
type_to_exclude = "folder"
old_count = len(items)
if hide_drm_protected:
Logger.debug("Hiding DRM items")
items = [
i for i in items
if not i.isDrmProtected or i.type == type_to_exclude
]
if hide_geo_locked:
Logger.debug("Hiding GEO Locked items due to GEO region: %s",
self.language)
items = [
i for i in items
if not i.isGeoLocked or i.type == type_to_exclude
]
if hide_premium:
Logger.debug("Hiding Premium items")
items = [
i for i in items if not i.isPaid or i.type == type_to_exclude
]
# Local import for performance
from resources.lib.cloaker import Cloaker
cloaker = Cloaker(self,
AddonSettings.store(LOCAL),
logger=Logger.instance())
if not AddonSettings.show_cloaked_items():
Logger.debug("Hiding Cloaked items")
items = [i for i in items if not cloaker.is_cloaked(i.url)]
else:
cloaked_items = [i for i in items if cloaker.is_cloaked(i.url)]
for c in cloaked_items:
c.isCloaked = True
if len(items) != old_count:
Logger.info(
"Hidden %s items due to DRM/GEO/Premium/cloak filter (Hide Folders=%s)",
old_count - len(items), hide_folders)
# Check for grouping or not
limit = AddonSettings.get_list_limit()
folder_items = [i for i in items if i.type.lower() == "folder"]
# we should also de-duplicate before calculating
folder_items = list(set(folder_items))
folders = len(folder_items)
if 0 < limit < folders:
# let's filter them by alphabet if the number is exceeded
Logger.debug(
"Creating Groups for list exceeding '%s' folder items. Total folders found '%s'.",
limit, folders)
other = "\a{}".format(
LanguageHelper.get_localized_string(LanguageHelper.OtherChars))
title_format = "\a{}".format(
LanguageHelper.get_localized_string(LanguageHelper.StartWith))
result = dict()
non_grouped = []
# Should we remove prefixes just as Kodi does?
# prefixes = ("de", "het", "the", "een", "a", "an")
for sub_item in items:
if sub_item.dontGroup or sub_item.type != "folder":
non_grouped.append(sub_item)
continue
char = sub_item.name[0].upper()
if char == "&":
title = HtmlEntityHelper.convert_html_entities(
sub_item.name)
char = title[0].upper()
# Should we de-prefix?
# for p in prefixes:
# if sub_item.name.lower().startswith(p + " "):
# char = sub_item.name[len(p) + 1][0].upper()
if char.isdigit():
char = "0-9"
elif not char.isalpha():
char = other
if char not in result:
Logger.trace("Creating Grouped item from: %s", sub_item)
if char == other:
item = MediaItem(
title_format.replace("'", "") % (char, ), "")
else:
item = MediaItem(title_format % (char.upper(), ), "")
item.complete = True
# item.set_date(2100 + ord(char[0]), 1, 1, text='')
result[char] = item
else:
item = result[char]
item.items.append(sub_item)
items = non_grouped + list(result.values())
# In order to get a better performance in de-duplicating and keeping the sort order
# we first need to store the order in a lookup table. Then we use sorted(set()) and
# use that lookup table for sorting. Using sorted(set(), items.index) this will be
# an O(n) (for the index()) times O(n*log(n)) (for the sorted) = O(n^2*log(n)!.
# The dictionary lookup (O(1)) saves us an O(n).
# See https://wiki.python.org/moin/TimeComplexity
sorted_order = {}
for i in range(0, len(items)):
sorted_order[items[i]] = i
unique_results = sorted(set(items), key=sorted_order.get)
Logger.trace("Found '%d' items of which '%d' are unique.", len(items),
len(unique_results))
return unique_results
def extract_hero_data(self, data):
""" Extacts the Hero json data
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
hero_data = Regexer.do_regex(r'data-hero="([^"]+)', data)[0]
hero_data = HtmlEntityHelper.convert_html_entities(hero_data)
Logger.trace(hero_data)
hero_json = JsonHelper(hero_data)
hero_playlists = hero_json.get_value("data", "playlists")
if not hero_playlists:
# set an empty object
hero_json.json = {}
current = self.parentItem.metaData.get("current_playlist", None)
if current == "clips":
Logger.debug("Found 'clips' metadata, only listing clips")
hero_json.json = {}
return hero_json, items
if current is None:
# Add clips folder
clip_title = LanguageHelper.get_localized_string(
LanguageHelper.Clips)
clips = MediaItem("\a.: %s :." % (clip_title, ),
self.parentItem.url)
clips.metaData[self.__meta_playlist] = "clips"
self.__no_clips = True
items.append(clips)
# See if there are seasons to show
if len(hero_playlists) == 1:
# first items, list all, except if there is only a single season
Logger.debug("Only one folder playlist found. Listing that one")
return hero_json, items
if current is None:
# list all folders
for playlist in hero_playlists:
folder = self.create_folder_item(playlist)
items.append(folder)
# clear the json item to prevent further listing
hero_json.json = {}
return hero_json, items
# list the correct folder
current_list = [l for l in hero_playlists if l["id"] == current]
if current_list:
# we are listing a subfolder, put that one on index 0 and then also
hero_playlists.insert(0, current_list[0])
self.__no_clips = True
Logger.debug("Pre-Processing finished")
return hero_json, items
def __convert_dc_subtitle_to_srt(dc_subtitle):
"""Converts DC Subtitle format into SRT format:
Arguments:
dcSubtitle : string - DC Subtitle subtitle format
Returns:
SRT formatted subtitle:
Example:
<Subtitle SpotNumber="1" TimeIn="00:00:01:220" TimeOut="00:00:04:001" FadeUpTime="20" FadeDownTime="20">
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="6.0">Line 1</Text>
</Subtitle>
<Subtitle SpotNumber="2" TimeIn="00:02:07:180" TimeOut="00:02:10:040" FadeUpTime="20" FadeDownTime="20">
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="6.0">Line 1</Text>
</Subtitle>
<Subtitle SpotNumber="3" TimeIn="00:02:15:190" TimeOut="00:02:17:190" FadeUpTime="20" FadeDownTime="20">
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="14.0">Line 1</Text>
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="6.0">Line 2</Text>
</Subtitle>
<Subtitle SpotNumber="4" TimeIn="00:03:23:140" TimeOut="00:03:30:120" FadeUpTime="20" FadeDownTime="20">
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="14.0">Line 1</Text>
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="14.0">Line 2</Text>
<Text Direction="horizontal" HAlign="center" HPosition="0.0" VAlign="bottom" VPosition="14.0">Line 3</Text>
</Subtitle>
Returns
1
00:00:20,000 --> 00:00:24,400
text
The format of the timecode is Hours:Minutes:Seconds:Ticks where a "Tick"
is a value of between 0 and 249 and lasts 4 milliseconds.
"""
parse_regex = r'<subtitle[^>]+spotnumber="(\d+)" timein="(\d+:\d+:\d+):(\d+)" ' \
r'timeout="(\d+:\d+:\d+):(\d+)"[^>]+>|<text[^>]+>([^<]+)</text>'
parse_regex = parse_regex.replace('"', '["\']')
subs = Regexer.do_regex(parse_regex, dc_subtitle)
srt = ""
i = 1
text = ""
start = ""
end = ""
for sub in subs:
#Logger.Trace(sub)
try:
if sub[0]:
# new start of a sub
if text and start and end:
# if we have a complete old one, save it
text = HtmlEntityHelper.convert_html_entities(text)
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end,
text.strip())
i += 1
start = "%s,%03d" % (sub[1], int(sub[2]))
end = "%s,%03d" % (sub[3], int(sub[4]))
text = ""
else:
text = "%s\n%s" % (text, sub[5].replace("<br />", "\n"))
except:
Logger.error("Error parsing subtitle: %s", sub, exc_info=True)
return srt
