def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
try:
if request.method == "GET":
response = self.__get_cached_response(request)
if response:
self.cache_store.cacheHits += 1
return response
except:
Logger.error("Error retrieving cache for %s", request.url, exc_info=True)
# Actually send a request
Logger.debug("Retrieving data from: %s", request.url)
response = super(CacheHTTPAdapter, self).send(request, stream, timeout, verify, cert, proxies)
try:
# Cache it if it was a cacheable response
cache_data = self.__extract_cache_data(response.headers)
if self.__should_cache(response, cache_data):
self.__store_response(request, response, cache_data)
if response.status_code == 304:
Logger.debug("304 Response found. Prolonging the %s", response.url)
self.cache_store.cacheHits += 1
response = self.__get_cached_response(request, no_check=True)
except:
Logger.error("Error storing cache for %s", request.url, exc_info=True)
return response
def __update_live_video(self, item, manifest, headers):
video_info = manifest.get_value("playable", "assets", 0)
url = video_info["url"]
encrypted = video_info["encrypted"]
part = item.create_new_empty_media_part()
if encrypted:
use_adaptive = AddonSettings.use_adaptive_stream_add_on(with_encryption=True)
if not use_adaptive:
Logger.error("Cannot playback encrypted item without inputstream.adaptive with encryption support")
return item
stream = part.append_media_stream(url, 0)
key = M3u8.get_license_key("", key_headers=headers, key_type="R")
M3u8.set_input_stream_addon_input(stream, proxy=self.proxy, headers=headers, license_key=key)
item.complete = True
else:
use_adaptive = AddonSettings.use_adaptive_stream_add_on(with_encryption=False)
if use_adaptive:
stream = part.append_media_stream(url, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy, headers=headers)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy, headers=headers):
item.complete = True
part.append_media_stream(s, b)
return item
def set_input_stream_addon_input(strm,
proxy=None,
headers=None,
license_key=None,
license_type="com.widevine.alpha",
max_bit_rate=None,
persist_storage=False,
service_certificate=None,
manifest_update=None):
""" Updates an existing stream with parameters for the inputstream adaptive add-on.
:param strm: (MediaStream) the MediaStream to update
:param proxy: (Proxy) The proxy to use for opening
:param dict headers: Possible HTTP Headers
:param str license_key: The value of the license key request
:param str license_type: The type of license key request used (see below)
:param int max_bit_rate: The maximum bitrate to use (optional)
:param bool persist_storage: Should we store certificates? And request server certificates?
:param str service_certificate: Use the specified server certificate
:returns: The updated stream
:rtype: MediaStream
Can be used like this:
part = item.create_new_empty_media_part()
stream = part.append_media_stream(m3u8url, 0)
M3u8.set_input_stream_addon_input(stream, self.proxy, self.headers)
item.complete = True
if maxBitRate is not set, the bitrate will be configured via the normal generic Retrospect
or channel settings.
"""
if license_key is not None:
# Local import to make sure the overhead is low
import inputstreamhelper
from resources.lib.logger import Logger
is_helper = inputstreamhelper.Helper('mpd', drm=license_type)
if is_helper.check_inputstream():
Logger.info(
"Widevine library was already installed or installed successfully."
)
else:
Logger.error(
"Widevine was not installed or failed to install.")
return Adaptive.set_input_stream_addon_input(
strm,
proxy,
headers,
manifest_type="mpd",
license_key=license_key,
license_type=license_type,
max_bit_rate=max_bit_rate,
persist_storage=persist_storage,
service_certificate=service_certificate,
manifest_update=manifest_update)
def __do_progress_callback(self, progress_callback, retrieved_size, total_size, completed):
""" Performs a callback, if the progressCallback was specified.
:param progress_callback: The callback method
:param retrieved_size: Number of bytes retrieved
:param total_size: Total number of bytes
:param completed: Are we done?
@rtype : Boolean Should we cancel the download?
"""
if progress_callback is None:
# no callback so it was not cancelled
return False
# calculated some stuff
self.__animationIndex = (self.__animationIndex + 1) % 4
bytes_to_mb = 1048576
animation_frames = ["-", "\\", "|", "/"]
animation = animation_frames[self.__animationIndex]
retrievedsize_mb = 1.0 * retrieved_size / bytes_to_mb
totalsize_mb = 1.0 * total_size / bytes_to_mb
if total_size > 0:
percentage = 100.0 * retrieved_size / total_size
else:
percentage = 0
status = '%s - %i%% (%.1f of %.1f MB)' % \
(animation, percentage, retrievedsize_mb, totalsize_mb)
try:
return progress_callback(retrieved_size, total_size, percentage, completed, status)
except:
Logger.error("Error in Progress Callback", exc_info=True)
# cancel the download
return True
def __html_entity_converter(entity):
"""Substitutes an HTML entity with the correct character
:param re.MatchObject entity: Value of the HTML entity without the '&'
:rtype: str
:return: Replaces &#xx where 'x' is single digit, or &...; where '.' is a
character into the real character. That character is returned.
"""
# Logger.Debug("1:%s, 2:%s", entity.group(1), entity.group(2))
try:
if entity.group(1) == "#":
# Logger.Trace("%s: %s", entity.group(2), chr(int(entity.group(2))))
return unichr(int(entity.group(2), 10))
elif entity.group(1) == "#x":
# check for hex values
return unichr(int(entity.group(2), 16))
elif entity.group(2) == 'apos':
# this one is not covert in name2codepoint
return "'"
else:
# Logger.Trace("%s: %s", entity.group(2), htmldefs.name2codepoint[entity.group(2)])
return unichr(htmldefs.name2codepoint[entity.group(2)])
except:
Logger.error("Error converting HTMLEntities: &%s%s", entity.group(1), entity.group(2), exc_info=True)
return '&%s%s;' % (entity.group(1), entity.group(2))
def widevine_lib(self):
""" Retrieve the path of the Widevine libraries.
:return: The full path to either libwidevinecdm.so, widevinecdm.dll or libwidevinecdm.dylib
:rtype: str
"""
try:
input_stream_adaptive_id = 'inputstream.adaptive'
if not xbmc.getCondVisibility('System.HasAddon("{}")'.format(input_stream_adaptive_id)):
return "<no-addon>"
addon = xbmcaddon.Addon(input_stream_adaptive_id)
decrypter_path_from_settings = addon.getSetting('DECRYPTERPATH')
if decrypter_path_from_settings:
cdm_path = translatePath(decrypter_path_from_settings)
else:
cdm_path = os.path.join(translatePath("special://home/"), "cdm")
if not os.path.isdir(cdm_path):
return "<none>"
Logger.debug("Found CDM folder: %s", cdm_path)
widevine_libs = [f for f in os.listdir(cdm_path) if f in
("libwidevinecdm.so", "widevinecdm.dll", "libwidevinecdm.dylib")]
return os.path.join(cdm_path, widevine_libs[0]) if len(widevine_libs) == 1 else "<none>"
except:
Logger.error("Error determining Widevine lib path.", exc_info=True)
return "<error>"
def create_folder_item(self, result_set):
""" Creates a MediaItem of type 'folder' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'folder'.
:rtype: MediaItem|None
"""
if len(result_set) > 3 and result_set[3] != "":
Logger.debug("Sub category folder found.")
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[3]))
name = "\a.: %s :." % (result_set[4], )
item = MediaItem(name, url)
item.complete = True
item.type = "folder"
return item
url = parse.urljoin(
self.baseUrl,
HtmlEntityHelper.convert_html_entities(result_set[0]))
name = HtmlEntityHelper.convert_html_entities(result_set[1])
helper = HtmlHelper(result_set[2])
description = helper.get_tag_content("div", {'class': 'description'})
item = MediaItem(name, "%s/RSS" % (url, ))
item.type = 'folder'
item.description = description.strip()
date = helper.get_tag_content("div", {'class': 'date'})
if date == "":
date = helper.get_tag_content("span",
{'class': 'lastPublishedDate'})
if not date == "":
date_parts = Regexer.do_regex(r"(\w+) (\d+)[^<]+, (\d+)", date)
if len(date_parts) > 0:
date_parts = date_parts[0]
month_part = date_parts[0].lower()
day_part = date_parts[1]
year_part = date_parts[2]
try:
month = DateHelper.get_month_from_name(month_part, "en")
item.set_date(year_part, month, day_part)
except:
Logger.error("Error matching month: %s",
month_part,
exc_info=True)
item.complete = True
return item
def update_live_channel(self, item):
""" Updates an existing live stream MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url, no_cache=True)
manifest = JsonHelper(data)
if "nonPlayable" in manifest.json and manifest.json["nonPlayable"]:
Logger.error("Cannot update Live: %s", item)
return item
source = manifest.get_value("sourceMedium")
if source == "audio":
return self.__update_live_audio(item, manifest)
else:
return self.__update_live_video(item, manifest)
def __convert_ttml_to_srt(ttml):
"""Converts sami format into SRT format:
Arguments:
ttml : string - TTML (Timed Text Markup Language) subtitle format
Returns:
SRT formatted subtitle:
Example:
1
00:00:20,000 --> 00:00:24,400
text
"""
pars_regex = r'<p[^>]+begin="([^"]+)\.(\d+)"[^>]+end="([^"]+)\.(\d+)"[^>]*>([\w\W]+?)</p>'
subs = Regexer.do_regex(pars_regex, ttml)
srt = ""
i = 1
for sub in subs:
try:
start = "%s,%03d" % (sub[0], int(sub[1]))
end = "%s,%03d" % (sub[2], int(sub[3]))
text = sub[4].replace("<br />", "\n")
text = HtmlEntityHelper.convert_html_entities(text)
text = text.replace("\r\n", "")
srt = "%s\n%s\n%s --> %s\n%s\n" % (srt, i, start, end, text.strip())
i += 1
except:
Logger.error("Error parsing subtitle: %s", sub[1], exc_info=True)
return srt
def on_action_from_context_menu(self, action):
"""Peforms the action from a custom contextmenu
Arguments:
action : String - The name of the method to call
"""
Logger.debug("Performing Custom Contextmenu command: %s", action)
item = self.media_item
if not item.complete:
Logger.debug(
"The contextmenu action requires a completed item. Updating %s",
item)
item = self.channelObject.process_video_item(item)
if not item.complete:
Logger.warning(
"update_video_item returned an item that had item.complete = False:\n%s",
item)
# invoke
function_string = "returnItem = self.channelObject.%s(item)" % (
action, )
Logger.debug("Calling '%s'", function_string)
try:
# noinspection PyRedundantParentheses
exec(function_string) # NOSONAR We just need this here.
except:
Logger.error("on_action_from_context_menu :: Cannot execute '%s'.",
function_string,
exc_info=True)
return
def __fetch_textures(self):
textures_to_retrieve = TextureHandler.instance(
).number_of_missing_textures()
if textures_to_retrieve > 0:
w = None
try:
# show a blocking or background progress bar
if textures_to_retrieve > 4:
w = XbmcDialogProgressWrapper(
"%s: %s" % (Config.appName,
LanguageHelper.get_localized_string(
LanguageHelper.InitChannelTitle)),
LanguageHelper.get_localized_string(
LanguageHelper.FetchTexturesTitle),
# Config.textureUrl
)
else:
w = XbmcDialogProgressBgWrapper(
"%s: %s" % (Config.appName,
LanguageHelper.get_localized_string(
LanguageHelper.FetchTexturesTitle)),
Config.textureUrl)
TextureHandler.instance().fetch_textures(w.progress_update)
except:
Logger.error("Error fetching textures", exc_info=True)
finally:
if w is not None:
# always close the progress bar
w.close()
return
def add(self, channel, item, action_url):
""" Adds a favourite for a specific channel.
:param channel: The channel
:param item: The mediaitem
:param str action_url: The mediaitem's actionUrl
"""
Logger.debug("Adding item %s\nfor channel %s\n%s", item, channel,
action_url)
file_name = self.__filePattern % (channel.guid, item.guid)
file_path = os.path.join(self.FavouriteFolder, file_name)
pickle = self.__pickler.pickle_media_item(item)
# Just double check for folder existence
if not os.path.isdir(self.FavouriteFolder):
os.makedirs(self.FavouriteFolder)
# replacing to pickle in the actionUrl to save space
action_url = self.__remove_pickle(action_url)
try:
with io.open(file_path, mode='w', encoding='utf-8') as file_handle:
file_handle.write(
"%s\n%s\n%s\n%s" %
(channel.channelName, item.name, action_url, pickle))
except:
Logger.error("Error saving favourite", exc_info=True)
raise
return
def __update_add_on_settings_with_channel_selection(contents, channels):
""" Adds the settings part that allows the selection of the channel for which the channel settings should
be displayed.
:param str contents: The current settings
:param list[Any] channels: The available channels
:return: updated contents
:rtype: str
"""
if "<!-- start of active channels -->" not in contents:
Logger.error("No '<!-- start of active channels -->' found in settings.xml. Stopping updating.")
return
# Create new XML
channel_selection_xml = ' <!-- start of active channels -->\n' \
' <setting id="config_channel" type="select" label="30040" values="'
channel_safe_names = "|".join([c.safe_name for c in channels])
channel_selection_xml = "%s%s" % (channel_selection_xml, channel_safe_names)
channel_selection_xml = '%s" />' % (channel_selection_xml.rstrip("|"),)
# replace the correct parts
begin = contents[:contents.find('<!-- start of active channels -->')].strip()
end = contents[contents.find('<!-- end of active channels -->'):].strip()
contents = "%s\n%s\n %s" % (begin, channel_selection_xml, end)
return contents
def execute(self):
Logger.debug("Performing Custom Contextmenu command: %s",
self.__action)
item = self.__media_item
if not item.complete:
Logger.debug(
"The contextmenu action requires a completed item. Updating %s",
item)
item = self.__channel.process_video_item(item)
if not item.complete:
Logger.warning(
"update_video_item returned an item that had item.complete = False:\n%s",
item)
# invoke the call
function_string = "returnItem = channel_object.%s(item)" % (
self.__action, )
Logger.debug("Calling '%s'", function_string)
try:
# noinspection PyRedundantParentheses
exec(function_string) # NOSONAR We just need this here.
except:
Logger.error("on_action_from_context_menu :: Cannot execute '%s'.",
function_string,
exc_info=True)
return
def get_setting(self, setting_id):
""" Retrieves an encrypted setting from the Kodi Add-on Settings.
:param str setting_id: the ID for the setting to retrieve.
:return: the decrypted value for the setting.
:rtype: str
"""
Logger.info("Decrypting value for setting '%s'", setting_id)
encrypted_value = AddonSettings.get_setting(setting_id)
if not encrypted_value:
Logger.warning("Found empty string as encrypted data")
return encrypted_value
try:
decrypted_value = self.__decrypt(encrypted_value, Vault.__Key)
if not decrypted_value.startswith(setting_id):
Logger.error("Invalid decrypted value for setting '%s'",
setting_id)
return None
decrypted_value = decrypted_value[len(setting_id) + 1:]
Logger.info("Successfully decrypted value for setting '%s'",
setting_id)
except UnicodeDecodeError:
Logger.error(
"Invalid Unicode data returned from decryption. Must be wrong data"
)
return None
return decrypted_value
def __send_log(self):
""" Send log files via Pastbin or Gist. """
from resources.lib.helpers.logsender import LogSender
sender_mode = 'hastebin'
log_sender = LogSender(Config.logSenderApi,
logger=Logger.instance(),
mode=sender_mode)
try:
title = LanguageHelper.get_localized_string(
LanguageHelper.LogPostSuccessTitle)
url_text = LanguageHelper.get_localized_string(
LanguageHelper.LogPostLogUrl)
files_to_send = [
Logger.instance().logFileName,
Logger.instance().logFileName.replace(".log", ".old.log")
]
if sender_mode != "gist":
paste_url = log_sender.send_file(Config.logFileNameAddon,
files_to_send[0])
else:
paste_url = log_sender.send_files(Config.logFileNameAddon,
files_to_send)
XbmcWrapper.show_dialog(title, url_text % (paste_url, ))
except Exception as e:
Logger.error("Error sending %s",
Config.logFileNameAddon,
exc_info=True)
title = LanguageHelper.get_localized_string(
LanguageHelper.LogPostErrorTitle)
error_text = LanguageHelper.get_localized_string(
LanguageHelper.LogPostError)
error = error_text % (str(e), )
XbmcWrapper.show_dialog(title, error.strip(": "))
def get_channel(self, class_name, channel_code, info_only=False):
""" Fetches a single channel for a given className and channelCode
If updated channels are found, the those channels are indexed and the
channel index is rebuild.
:param str|unicode class_name: The chn_<name> class name.
:param str|unicode channel_code: A possible channel code within the channel set.
:param bool info_only: Only return the ChannelInfo.
:return: a Channel object
:rtype: Channel
"""
channel_set = self.__channelIndex[self.__CHANNEL_INDEX_CHANNEL_KEY].get(class_name, None)
if channel_set is None:
Logger.error("Could not find info for channelClass '%s'.", class_name)
return None
channel_set_info_path = channel_set[self.__CHANNEL_INDEX_CHANNEL_INFO_KEY]
channel_set_version = channel_set[self.__CHANNEL_INDEX_CHANNEL_VERSION_KEY]
if not os.path.isfile(channel_set_info_path) and not self.__reindexed:
Logger.warning("Missing channel_set file: %s.", channel_set_info_path)
self.__rebuild_index()
return self.get_channel(class_name, channel_code)
channel_infos = ChannelInfo.from_json(channel_set_info_path, channel_set_version)
if channel_code is None:
channel_infos = [ci for ci in channel_infos if ci.channelCode is None]
else:
channel_infos = [ci for ci in channel_infos if ci.channelCode == channel_code]
if len(channel_infos) != 1:
Logger.error("Found none or more than 1 matches for '%s' and '%s' in the channel index.",
class_name, channel_code or "None")
return None
else:
Logger.debug("Found single channel in the channel index: %s.", channel_infos[0])
if self.__is_channel_set_updated(channel_infos[0]):
# let's see if the index has already been updated this section, of not, do it and
# restart the ChannelRetrieval.
if not self.__reindexed:
# rebuild and restart
Logger.warning("Re-index channel index due to channel_set update: %s.", channel_set_info_path)
self.__rebuild_index()
else:
Logger.warning("Found updated channel_set: %s.", channel_set_info_path)
# new we should init all channels by loading them all, just to be sure that all is ok
Logger.debug("Going to fetching all channels to init them all.")
self.get_channels()
return self.get_channel(class_name, channel_code)
if info_only:
return channel_infos[0]
return channel_infos[0].get_channel()
def update_part_with_m3u8_streams(part,
url,
encrypted=False,
headers=None,
map_audio=False,
bitrate=0,
channel=None):
""" Updates an existing MediaItemPart with M3u8 data either using the Adaptive Inputstream
Add-on or with the built-in code.
:param MediaItemPart part: The part to update
:param str url: The url to download
:param bool encrypted: Is the stream encrypted?
:param dict[str,str] headers: Possible HTTP Headers
:param bool map_audio: Should audio tracks be mapped separately?
:param int bitrate: Initial bitrate to use. Will be overridden later.
:param ChannelInfo channel: If specified, the channel specific configuration is
considered.
:return: indication if updating was successful.
:rtype: bool
"""
input_stream = AddonSettings.use_adaptive_stream_add_on(
encrypted, channel=channel)
if not input_stream and encrypted:
Logger.error(
"Cannot play encrypted stream without InputStream Adaptive with Encryption support!"
)
return False
if input_stream:
Logger.debug(
"Using InputStream Adaptive add-on for M3u8 playback.")
stream = part.append_media_stream(url, bitrate)
M3u8.set_input_stream_addon_input(stream, headers)
return True
complete = False
if map_audio:
Logger.debug(
"Using Retrospect code with Audio mapping for M3u8 playback.")
for s, b, a in M3u8.get_streams_from_m3u8(url, map_audio=True):
if a:
audio_part = a.rsplit("-", 1)[-1]
audio_part = "-%s" % (audio_part, )
s = s.replace(".m3u8", audio_part)
part.append_media_stream(s, b)
complete = True
else:
Logger.debug("Using Retrospect code for M3u8 playback.")
for s, b in M3u8.get_streams_from_m3u8(url):
part.append_media_stream(s, b)
complete = True
return complete
def __init__(self, title, url, type="folder"):
""" Creates a new MediaItem.
The `url` can contain an url to a site more info about the item can be
retrieved, for instance for a video item to retrieve the media url, or
in case of a folder where child items can be retrieved.
Essential is that no encoding (like UTF8) is specified in the title of
the item. This is all taken care of when creating Kodi items in the
different methods.
:param str|unicode title: The title of the item, used for appearance in lists.
:param str|unicode url: Url that used for further information retrieval.
:param str type: Type of MediaItem (folder, video, audio). Defaults to 'folder'.
"""
name = title.strip()
self.name = name
self.url = url
self.actionUrl = None
self.MediaItemParts = []
self.description = ""
self.thumb = "" # : The local or remote image for the thumbnail of episode
self.fanart = "" # : The fanart url
self.icon = "" # : low quality icon for list
self.__date = "" # : value show in interface
self.__timestamp = datetime.min # : value for sorting, this one is set to minimum so if non is set, it's shown at the bottom
self.__expires_datetime = None # : datetime value of the expire time
self.type = type # : video, audio, folder, append, page, playlist
self.dontGroup = False # : if set to True this item will not be auto grouped.
self.isLive = False # : if set to True, the item will have a random QuerySting param
self.isGeoLocked = False # : if set to True, the item is GeoLocked to the channels language (o)
self.isDrmProtected = False # : if set to True, the item is DRM protected and cannot be played (^)
self.isPaid = False # : if set to True, the item is a Paid item and cannot be played (*)
self.__infoLabels = dict() # : Additional Kodi InfoLabels
self.complete = False
self.items = []
self.HttpHeaders = dict() # : http headers for the item data retrieval
# Items that are not essential for pickled
self.isCloaked = False
self.metaData = dict() # : Additional data that is for internal / routing use only
# GUID used for identification of the object. Do not set from script, MD5 needed
# to prevent UTF8 issues
try:
self.guid = "%s%s" % (EncodingHelper.encode_md5(title), EncodingHelper.encode_md5(url or ""))
except:
Logger.error("Error setting GUID for title:'%s' and url:'%s'. Falling back to UUID", title, url, exc_info=True)
self.guid = self.__get_uuid()
self.guidValue = int("0x%s" % (self.guid,), 0)
def log_on(self):
""" Logs on to a website, using an url.
First checks if the channel requires log on. If so and it's not already
logged on, it should handle the log on. That part should be implemented
by the specific channel.
More arguments can be passed on, but must be handled by custom code.
After a successful log on the self.loggedOn property is set to True and
True is returned.
:return: indication if the login was successful.
:rtype: bool
"""
if self.__idToken:
return True
# check if there is a refresh token
# refresh token: viervijfzes_refresh_token
refresh_token = AddonSettings.get_setting("viervijfzes_refresh_token")
client = AwsIdp("eu-west-1_dViSsKM5Y",
"6s1h851s8uplco5h6mqh1jac8m",
proxy=self.proxy,
logger=Logger.instance())
if refresh_token:
id_token = client.renew_token(refresh_token)
if id_token:
self.__idToken = id_token
return True
else:
Logger.info("Extending token for VierVijfZes failed.")
# username: viervijfzes_username
username = AddonSettings.get_setting("viervijfzes_username")
# password: viervijfzes_password
v = Vault()
password = v.get_setting("viervijfzes_password")
if not username or not password:
XbmcWrapper.show_dialog(
title=None,
lines=LanguageHelper.get_localized_string(
LanguageHelper.MissingCredentials),
)
return False
id_token, refresh_token = client.authenticate(username, password)
if not id_token or not refresh_token:
Logger.error("Error getting a new token. Wrong password?")
return False
self.__idToken = id_token
AddonSettings.set_setting("viervijfzes_refresh_token", refresh_token)
return True
def create_video_item(self, result_set):
""" Creates a MediaItem of type 'video' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
If the item is completely processed an no further data needs to be fetched
the self.complete property should be set to True. If not set to True, the
self.update_video_item method is called if the item is focussed or selected
for playback.
:param list[str]|dict[str,str] result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'video' or 'audio' (despite the method's name).
:rtype: MediaItem|None
"""
# Logger.Trace(result_set)
xml_data = XmlHelper(result_set)
title = xml_data.get_single_node_content("title")
url = xml_data.get_single_node_content("link")
description = xml_data.get_single_node_content("description")
description = description.replace("<![CDATA[ ",
"").replace("]]>", "").replace(
"<p>", "").replace("</p>", "\n")
item = MediaItem(title, url)
item.type = 'video'
item.complete = False
item.description = description
item.thumb = self.noImage
item.icon = self.icon
date = xml_data.get_single_node_content("pubDate")
date_result = Regexer.do_regex(r"\w+, (\d+) (\w+) (\d+)", date)[-1]
day = date_result[0]
month_part = date_result[1].lower()
year = date_result[2]
try:
month_lookup = [
"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep",
"oct", "nov", "dec"
]
month = month_lookup.index(month_part) + 1
item.set_date(year, month, day)
except:
Logger.error("Error matching month: %s",
result_set[4].lower(),
exc_info=True)
return item
def __extract_session_data(self, logon_data):
logon_json = JsonHelper(logon_data)
result_code = logon_json.get_value("statusCode")
if result_code != 200:
Logger.error("Error loging in: %s - %s", logon_json.get_value("errorMessage"),
logon_json.get_value("errorDetails"))
return None, None, None
return logon_json.get_value("UID"), \
logon_json.get_value("UIDSignature"), \
logon_json.get_value("signatureTimestamp")
def log_on(self, username, password):
""" Peforms the logon of a user.
:param str username: The username
:param str password: The password to use
:returns: a AuthenticationResult with the result of the log on
:rtype: AuthenticationResult
"""
# first we need a random context_id R<10 numbers>
context_id = int(random.random() * 8999999999) + 1000000000
# then we do an initial bootstrap call, which retrieves the `gmid` and `ucid` cookies
url = "https://sso.rtl.nl/accounts.webSdkBootstrap" \
"?apiKey={}" \
"&pageURL=https%3A%2F%2Fwww.rtlxl.nl%2F" \
"&format=json" \
"&callback=gigya.callback" \
"&context=R{}".format(self.api_key, context_id)
init_login = UriHandler.open(url, no_cache=True)
init_data = JsonHelper(init_login)
if init_data.get_value("statusCode") != 200:
Logger.error("Error initiating login")
return AuthenticationResult(None)
# actually do the login request, which requires an async call to retrieve the result
login_url = "https://sso.rtl.nl/accounts.login" \
"?context={0}".format(context_id)
login_data = {
"loginID": username,
"password": password,
# "include": "profile,data",
# "includeUserInfo": "true",
"pageURL": "https://www.rtlxl.nl/profiel",
"format": "json",
# "callback": "gigya.callback",
"context": "R{}".format(context_id),
"targetEnv": "jssdk",
"sessionExpiration": 7776000
}
login_data.update(self.__common_param_dict)
login_response = UriHandler.open(login_url,
data=login_data,
no_cache=True)
# Process the result
authentication_result = self.__extract_session_data(login_response)
authentication_result.existing_login = False
return authentication_result
def __requests(self, uri, proxy, params, data, json, referer,
additional_headers, no_cache, stream):
with requests.session() as s:
s.cookies = self.cookieJar
s.verify = not self.ignoreSslErrors
if self.cacheStore and not no_cache:
Logger.trace("Adding the %s to the request", self.cacheStore)
s.mount("https://", CacheHTTPAdapter(self.cacheStore))
s.mount("http://", CacheHTTPAdapter(self.cacheStore))
proxies = self.__get_proxies(proxy, uri)
if proxies is not None and "dns" in proxies:
s.mount("https://", DnsResolverHTTPAdapter(uri, proxies["dns"],
logger=Logger.instance()))
headers = self.__get_headers(referer, additional_headers)
if params is not None:
# Old UriHandler behaviour. Set form header to keep compatible
if "content-type" not in headers:
headers["content-type"] = "application/x-www-form-urlencoded"
Logger.info("Performing a POST with '%s' for %s", headers["content-type"], uri)
r = s.post(uri, data=params, proxies=proxies, headers=headers,
stream=stream, timeout=self.webTimeOut)
elif data is not None:
# Normal Requests compatible data object
Logger.info("Performing a POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri)
r = s.post(uri, data=data, proxies=proxies, headers=headers,
stream=stream, timeout=self.webTimeOut)
elif json is not None:
Logger.info("Performing a json POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri)
r = s.post(uri, json=json, proxies=proxies, headers=headers,
stream=stream, timeout=self.webTimeOut)
else:
Logger.info("Performing a GET for %s", uri)
r = s.get(uri, proxies=proxies, headers=headers,
stream=stream, timeout=self.webTimeOut)
if r.ok:
Logger.info("%s resulted in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason, r.elapsed, r.url)
else:
Logger.error("%s failed with '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason, r.elapsed, r.url)
self.status = UriStatus(code=r.status_code, url=r.url, error=not r.ok, reason=r.reason)
if self.cookieJarFile:
# noinspection PyUnresolvedReferences
self.cookieJar.save()
return r
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
if "index.html" in item.url:
data = UriHandler.open(
item.url,
additional_headers={"Referer": "http://www.{}/".format(self.__country_id)}
)
json_data = JsonHelper(data)
guid = json_data.get_value("feed", "items", 0, "guid")
url = "https://media-utils.mtvnservices.com/services/MediaGenerator/" \
"{}?arcStage=live&format=json&acceptMethods=hls&clang=nl" \
"&https=true".format(guid)
else:
url = item.url
data = UriHandler.open(url)
json_data = JsonHelper(data)
url = json_data.get_value("package", "video", "item", 0, "rendition", 0, "src")
if not url:
error = json_data.get_value("package", "video", "item", 0, "text")
Logger.error("Error resolving url: %s", error)
XbmcWrapper.show_dialog(LanguageHelper.ErrorId, error)
return item
item.MediaItemParts = []
part = item.create_new_empty_media_part()
part.append_media_stream(url, 0)
item.complete = True
return item
def get_channel(self, channel_id, channel_code, info_only=False):
""" Fetches a single channel for a given className and channelCode
If updated channels are found, the those channels are indexed and the
channel index is rebuild.
:param str|unicode channel_id: The chn_<name> class name.
:param str|unicode channel_code: A possible channel code within the channel set.
:param bool info_only: Only return the ChannelInfo.
:return: a Channel object
:rtype: Channel
"""
# determine the channel folder
channel_path = os.path.join(Config.rootDir, self.__INTERNAL_CHANNEL_PATH)
channel_pack, channel_set = channel_id.rsplit(".", 1)
channel_set_info_path = os.path.join(channel_path, channel_pack, channel_set, "chn_{}.json".format(channel_set))
channel_infos = ChannelInfo.from_json(channel_set_info_path)
if channel_code is None:
channel_infos = [ci for ci in channel_infos if ci.channelCode is None]
else:
channel_infos = [ci for ci in channel_infos if ci.channelCode == channel_code]
if len(channel_infos) != 1:
Logger.error("Found none or more than 1 matches for '%s' and '%s' in the channel index.",
channel_id, channel_code or "None")
return None
else:
Logger.debug("Found single channel in the channel index: %s.", channel_infos[0])
channel_info = channel_infos[0]
if self.__is_channel_set_updated(channel_info):
Logger.warning("Found updated channel_set: %s.", channel_set_info_path)
# new we should init all channels by loading them all, just to be sure that all is ok
Logger.debug("Going to fetching all channels to init them all.")
self.get_channels()
return self.get_channel(channel_id, channel_code)
if channel_info.ignore:
Logger.warning("Channel %s is ignored in channel set", channel_info)
return None
if info_only:
return channel_info
return channel_info.get_channel()
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
NOTE: This is a 100% copy of the chn_vtmbe.Channel.update_html_clip_item
"""
data = UriHandler.open(item.url)
json_data = Regexer.do_regex(
r"Drupal\.settings,\s*({[\w\W]+?})\);\s*//-->", data)
json_data = JsonHelper(json_data[-1])
video_info = json_data.get_value('medialaan_player', )
video_config = None
for key in video_info:
Logger.trace("Checking key: %s", key)
if "videoConfig" not in video_info[key]:
continue
video_config = video_info[key]['videoConfig']['video']
break
if not video_config:
Logger.error("No video info found.")
streams = video_config['formats']
for stream in streams:
stream_url = stream['url']
if stream['type'] == "mp4":
item.append_single_stream(stream_url, 0)
item.complete = True
return item
def header(self, uri, proxy=None, referer=None, additional_headers=None):
""" Retrieves header information only.
:param str uri: The URI to fetch the header from.
:param ProxyInfo|none proxy: The address and port (proxy.address.ext:port) of a
proxy server that should be used.
:param str|none referer: The http referer to use.
:param dict|none additional_headers: The optional headers.
:return: Content-type and the URL to which a redirect could have occurred.
:rtype: tuple[str,str]
"""
with requests.session() as s:
s.cookies = self.cookieJar
s.verify = not self.ignoreSslErrors
proxies = self.__get_proxies(proxy, uri)
headers = self.__get_headers(referer, additional_headers)
Logger.info("Performing a HEAD for %s", uri)
r = s.head(uri,
proxies=proxies,
headers=headers,
allow_redirects=True,
timeout=self.webTimeOut)
content_type = r.headers.get("Content-Type", "")
real_url = r.url
self.status = UriStatus(code=r.status_code,
url=uri,
error=not r.ok,
reason=r.reason)
if self.cookieJarFile:
# noinspection PyUnresolvedReferences
self.cookieJar.save()
if r.ok:
Logger.info("%s resulted in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
return content_type, real_url
else:
Logger.error("%s failed with in '%s %s' (%s) for %s",
r.request.method, r.status_code, r.reason,
r.elapsed, r.url)
return "", ""
def convert_html_entities(html):
"""Convert the HTML entities into their real characters
:param str|None html: The HTML to convert.
:return: The HTML with converted characters.
:rtype: str
"""
try:
return HtmlEntityHelper.__convert_html_entities(html)
except:
Logger.error("Error converting: %s", html, exc_info=True)
return html
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
if not item.url.endswith("m3u8"):
data = UriHandler.open(item.url, proxy=self.proxy)
json_data = Regexer.do_regex(self.mediaUrlRegex, data)
if not json_data:
Logger.error("Cannot find JSON stream info.")
return item
json = JsonHelper(json_data[0])
Logger.trace(json.json)
stream = json.get_value("source", "hls")
if stream is None:
stream = json.get_value("mzsource", "hls")
Logger.debug("Found HLS: %s", stream)
else:
stream = item.url
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(stream, self.proxy):
item.complete = True
part.append_media_stream(s, b)
# var playerConfig = {"id":"mediaplayer","width":"100%","height":"100%","autostart":"false","image":"http:\/\/www.ketnet.be\/sites\/default\/files\/thumb_5667ea22632bc.jpg","brand":"ketnet","source":{"hls":"http:\/\/vod.stream.vrt.be\/ketnet\/_definst_\/mp4:ketnet\/2015\/12\/Ben_ik_familie_van_R001_A0023_20151208_143112_864.mp4\/playlist.m3u8"},"analytics":{"type_stream":"vod","playlist":"Ben ik familie van?","program":"Ben ik familie van?","episode":"Ben ik familie van?: Warre - Aflevering 3","parts":"1","whatson":"270157835527"},"title":"Ben ik familie van?: Warre - Aflevering 3","description":"Ben ik familie van?: Warre - Aflevering 3"}
return item
