def UpdateVideoItem(self, item):
"""
Updates the item
"""
data = UriHandler.Open(item.url, proxy=self.proxy)
baseEncode = Regexer.DoRegex(self.mediaUrlRegex, data)[-1]
jsonData = EncodingHelper().DecodeBase64(baseEncode)
json = JsonHelper(jsonData, logger=Logger.Instance())
Logger.Trace(json)
# "flv": "http://media.dumpert.nl/flv/e2a926ff_10307954_804223649588516_151552487_n.mp4.flv",
# "tablet": "http://media.dumpert.nl/tablet/e2a926ff_10307954_804223649588516_151552487_n.mp4.mp4",
# "mobile": "http://media.dumpert.nl/mobile/e2a926ff_10307954_804223649588516_151552487_n.mp4.mp4",
item.MediaItemParts = []
part = item.CreateNewEmptyMediaPart()
streams = json.GetValue()
for key in streams:
if key == "flv":
part.AppendMediaStream(streams[key], 1000)
elif key == "tablet":
part.AppendMediaStream(streams[key], 800)
elif key == "mobile":
part.AppendMediaStream(streams[key], 450)
else:
Logger.Debug("Key '%s' was not used", key)
item.complete = True
Logger.Trace("VideoItem updated: %s", item)
return item
def __init__(self, channelInfo):
"""Initialisation of the class.
Arguments:
channelInfo: ChannelInfo - The channel info object to base this channel on.
All class variables should be instantiated here and this method should not
be overridden by any derived classes.
"""
chn_class.Channel.__init__(self, channelInfo)
# ============== Actual channel setup STARTS here and should be overwritten from derived classes ===============
self.noImage = "nosnlimage.png"
# setup the urls
# self.mainListUri = "http://nos.nl/"
self.mainListUri = "#getcategories"
# we need specific headers: APK:NosHttpClientHelper.java
salt = int(time.time())
# key = "%sRM%%j%%l@g@w_A%%" % (salt,)
# Logger.Trace("Found Salt: %s and Key: %s", salt, key)
# key = EncodingHelper.EncodeMD5(key, toUpper=False)
# self.httpHeaders = {"X-NOS-App": "Google/x86;Android/4.4.4;nl.nos.app/3.1",
# "X-NOS-Salt": salt,
# "X-NOS-Key": key}
userAgent = "%s;%d;%s/%s;Android/%s;nl.nos.app/%s" % (
"nos", salt, "Google", "Nexus", "6.0", "5.1.1")
string = ";UB}7Gaji==JPHtjX3@c%s" % (userAgent, )
string = EncodingHelper.EncodeMD5(string, toUpper=False).zfill(32)
xnos = string + base64.b64encode(userAgent)
self.httpHeaders = {"X-Nos": xnos}
self.baseUrl = "http://nos.nl"
# setup the main parsing data
self._AddDataParser(self.mainListUri, preprocessor=self.GetCategories)
self._AddDataParser(
"*",
# preprocessor=self.AddNextPage,
json=True,
parser=('items', ),
creator=self.CreateJsonVideo,
updater=self.UpdateJsonVideo)
self._AddDataParser("*",
json=True,
parser=('links', ),
creator=self.CreatePageItem)
#===============================================================================================================
# non standard items
# self.__IgnoreCookieLaw()
self.__pageSize = 50
# ====================================== Actual channel setup STOPS here =======================================
return
def __GetUUID(self):
"""Generates a Unique Identifier based on Time and Random Integers"""
t = long(time.time() * 1000)
r = long(random.random() * 100000000000000000L)
a = random.random() * 100000000000000000L
data = str(t) + ' ' + str(r) + ' ' + str(a)
data = EncodingHelper.EncodeMD5(data)
return data
def __init__(self):
""" Creates a new instance of the Vault class """
self.__newKeyGeneratedInConstructor = False # : This was the very first time a key was generated
# ask for PIN of no key is present
if Vault.__Key is None:
key = self.__get_application_key() # type: bytes
# was there a key? No, let's initialize it.
if key is None:
Logger.warning("No Application Key present. Intializing a new one.")
key = self.__get_new_key()
if not self.change_pin(key):
raise RuntimeError("Error creating Application Key.")
Logger.info("Created a new Application Key with MD5: %s (lengt=%s)",
EncodingHelper.encode_md5(key), len(key))
self.__newKeyGeneratedInConstructor = True
Vault.__Key = key
Logger.trace("Using Application Key with MD5: %s (lengt=%s)", EncodingHelper.encode_md5(key), len(key))
def __init__(self, channelInfo):
"""Initialisation of the class.
Arguments:
channelInfo: ChannelInfo - The channel info object to base this channel on.
All class variables should be instantiated here and this method should not
be overridden by any derived classes.
"""
chn_class.Channel.__init__(self, channelInfo)
# ============== Actual channel setup STARTS here and should be overwritten from derived classes ===============
self.noImage = "nosnlimage.png"
# setup the urls
# self.mainListUri = "http://nos.nl/"
self.mainListUri = "#getcategories"
# we need specific headers: APK:NosHttpClientHelper.java
salt = int(time.time())
key = "%sRM%%j%%l@g@w_A%%" % (salt,)
Logger.Trace("Found Salt: %s and Key: %s", salt, key)
key = EncodingHelper.EncodeMD5(key, toUpper=False)
self.httpHeaders = {"X-NOS-App": "Google/x86;Android/4.4.4;nl.nos.app/3.1",
"X-NOS-Salt": salt,
"X-NOS-Key": key}
self.baseUrl = "http://nos.nl"
# setup the main parsing data
self._AddDataParser(self.mainListUri, preprocessor=self.GetCategories)
self._AddDataParser("*", preprocessor=self.AddNextPage, json=True,
parser=(), creator=self.CreateJsonVideo, updater=self.UpdateJsonVideo)
self._AddDataParser("http://content.nos.nl/apps/feeds/most-watched-video/", json=True,
preprocessor=self.AddNextPage,
parser=('items',), creator=self.CreateJsonVideo, updater=self.UpdateJsonVideo)
#===============================================================================================================
# non standard items
# self.__IgnoreCookieLaw()
# ====================================== Actual channel setup STOPS here =======================================
return
def download_subtitle(url,
file_name="",
format='sami',
proxy=None,
replace=None):
"""Downloads a SAMI and stores the SRT in the cache folder
Arguments:
@param url: string - URL location of the SAMI file
@param file_name: string - [opt] Filename to use to store the subtitle in SRT format.
if not specified, an MD5 hash of the URL with .xml
extension will be used
@param format: string - Defines the source format. Defaults to Sami.
@param proxy: Proxy - If specified, a proxy will be used
@param replace: dict - Dictionary with key to will be replaced with their values
@return: The full patch of the cached SRT file.
"""
if file_name == "":
Logger.debug(
"No filename present, generating filename using MD5 hash of url."
)
file_name = "%s.srt" % (EncodingHelper.encode_md5(url), )
elif not file_name.endswith(".srt"):
Logger.debug("No SRT extension present, appending it.")
file_name = "%s.srt" % (file_name, )
srt = ""
try:
local_complete_path = os.path.join(Config.cacheDir, file_name)
# no need to download it again!
if os.path.exists(local_complete_path):
return local_complete_path
Logger.trace("Opening Subtitle URL")
raw = UriHandler.open(url, proxy=proxy)
if UriHandler.instance().status.error:
Logger.warning("Could not retrieve subtitle from %s", url)
return ""
if raw == "":
Logger.warning(
"Empty Subtitle path found. Not setting subtitles.")
return ""
# try to decode it as `raw` should be a string.
if isinstance(raw, bytes):
try:
raw = raw.decode()
except:
# fix some weird chars
try:
raw = raw.replace("\x96", "-")
except:
Logger.error("Error replacing some weird chars.")
Logger.warning(
"Converting input to UTF-8 using 'unicode_escape'")
raw = raw.decode('unicode_escape')
# do some auto detection
if raw.startswith("WEBVTT") and format != "webvtt":
Logger.info(
"Discovered subtitle format 'webvtt' instead of '%s'",
format)
format = "webvtt"
if format.lower() == 'sami':
srt = SubtitleHelper.__convert_sami_to_srt(raw)
elif format.lower() == 'srt':
srt = raw
elif format.lower() == 'webvtt':
srt = SubtitleHelper.__convert_web_vtt_to_srt(
raw) # With Krypton and Leia VTT is supported natively
elif format.lower() == 'ttml':
srt = SubtitleHelper.__convert_ttml_to_srt(raw)
elif format.lower() == 'dcsubtitle':
srt = SubtitleHelper.__convert_dc_subtitle_to_srt(raw)
elif format.lower() == 'json':
srt = SubtitleHelper.__convert_json_subtitle_to_srt(raw)
elif format.lower() == 'm3u8srt':
srt = SubtitleHelper.__convert_m3u8_srt_to_subtitle_to_srt(
raw, url, proxy)
else:
error = "Uknown subtitle format: %s" % (format, )
raise NotImplementedError(error)
if replace:
Logger.debug("Replacing SRT data: %s", replace)
for needle in replace:
srt = srt.replace(needle, replace[needle])
with io.open(local_complete_path, 'w', encoding="utf-8") as f:
f.write(srt)
Logger.info("Saved SRT as %s", local_complete_path)
return local_complete_path
except:
Logger.error("Error handling Subtitle file: [%s]",
srt,
exc_info=True)
return ""
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url, proxy=self.proxy)
json = JsonHelper(data, Logger.instance())
video_data = json.get_value("video")
if video_data:
part = item.create_new_empty_media_part()
if self.localIP:
part.HttpHeaders.update(self.localIP)
# get the videos
video_urls = video_data.get("videoReferences")
for video_url in video_urls:
# Logger.Trace(videoUrl)
stream_info = video_url['url']
if "manifest.f4m" in stream_info:
continue
elif "master.m3u8" in stream_info:
for s, b in M3u8.get_streams_from_m3u8(
stream_info, self.proxy, headers=part.HttpHeaders):
item.complete = True
part.append_media_stream(s, b)
# subtitles
subtitles = video_data.get("subtitleReferences")
if subtitles and subtitles[0]["url"]:
Logger.trace(subtitles)
sub_url = subtitles[0]["url"]
file_name = "%s.srt" % (EncodingHelper.encode_md5(sub_url), )
sub_data = UriHandler.open(sub_url, proxy=self.proxy)
# correct the subs
regex = re.compile(r"^1(\d:)", re.MULTILINE)
sub_data = re.sub(regex, r"0\g<1>", sub_data)
sub_data = re.sub(r"--> 1(\d):", r"--> 0\g<1>:", sub_data)
local_complete_path = os.path.join(Config.cacheDir, file_name)
Logger.debug("Saving subtitle to: %s", local_complete_path)
with open(local_complete_path, 'w') as f:
f.write(sub_data)
part.Subtitle = local_complete_path
item.complete = True
return item
def __init__(self, title, url, type="folder"):
""" Creates a new MediaItem.
The `url` can contain an url to a site more info about the item can be
retrieved, for instance for a video item to retrieve the media url, or
in case of a folder where child items can be retrieved.
Essential is that no encoding (like UTF8) is specified in the title of
the item. This is all taken care of when creating Kodi items in the
different methods.
:param str|unicode title: The title of the item, used for appearance in lists.
:param str|unicode url: Url that used for further information retrieval.
:param str type: Type of MediaItem (folder, video, audio). Defaults to 'folder'.
"""
name = title.strip()
self.name = name
self.url = url
self.actionUrl = None
self.MediaItemParts = []
self.description = ""
self.thumb = "" # : The local or remote image for the thumbnail of episode
self.fanart = "" # : The fanart url
self.icon = "" # : low quality icon for list
self.__date = "" # : value show in interface
self.__timestamp = datetime.datetime.min # : value for sorting, this one is set to minimum so if non is set, it's shown at the bottom
self.type = type # : video, audio, folder, append, page, playlist
self.dontGroup = False # : if set to True this item will not be auto grouped.
self.isLive = False # : if set to True, the item will have a random QuerySting param
self.isGeoLocked = False # : if set to True, the item is GeoLocked to the channels language (o)
self.isDrmProtected = False # : if set to True, the item is DRM protected and cannot be played (^)
self.isPaid = False # : if set to True, the item is a Paid item and cannot be played (*)
self.__infoLabels = dict() # : Additional Kodi InfoLabels
self.complete = False
self.items = []
self.HttpHeaders = dict() # : http headers for the item data retrieval
# Items that are not essential for pickled
self.isCloaked = False
self.metaData = dict(
) # : Additional data that is for internal / routing use only
# GUID used for identifcation of the object. Do not set from script, MD5 needed
# to prevent UTF8 issues
try:
self.guid = "%s%s" % (EncodingHelper.encode_md5(title),
EncodingHelper.encode_md5(url or ""))
except:
Logger.error(
"Error setting GUID for title:'%s' and url:'%s'. Falling back to UUID",
title,
url,
exc_info=True)
self.guid = self.__get_uuid()
self.guidValue = int("0x%s" % (self.guid, ), 0)
def DownloadSubtitle(url, fileName="", format='sami', proxy=None, replace=None):
"""Downloads a SAMI and stores the SRT in the cache folder
Arguments:
url : string - URL location of the SAMI file
Keyword Arguments:
fileName : string - Filename to use to store the subtitle in SRT format.
if not specified, an MD5 hash of the URL with .xml
extension will be used
format : string - defines the source format. Defaults to Sami.
Returns:
The full patch of the cached SRT file.
"""
if fileName == "":
Logger.Debug("No filename present, generating filename using MD5 hash of url.")
fileName = "%s.srt" % (EncodingHelper.EncodeMD5(url),)
elif not fileName.endswith(".srt"):
Logger.Debug("No SRT extension present, appending it.")
fileName = "%s.srt" % (fileName, )
srt = ""
try:
localCompletePath = os.path.join(Config.cacheDir, fileName)
# no need to download it again!
if os.path.exists(localCompletePath):
return localCompletePath
Logger.Trace("Opening Subtitle URL")
raw = UriHandler.Open(url, proxy=proxy)
if raw == "":
Logger.Warning("Empty Subtitle path found. Not setting subtitles.")
return ""
# try to decode it
try:
raw = raw.decode()
except:
# fix some weird chars
try:
raw = raw.replace("\x96", "-")
except:
Logger.Error("Error replacing some weird chars.")
Logger.Warning("Converting input to UTF-8 using 'unicode_escape'")
raw = raw.decode('unicode_escape')
if format.lower() == 'sami':
srt = SubtitleHelper.__ConvertSamiToSrt(raw)
elif format.lower() == 'srt':
srt = raw
elif format.lower() == 'webvtt':
srt = SubtitleHelper.__ConvertWebVttToSrt(raw)
elif format.lower() == 'ttml':
srt = SubtitleHelper.__ConvertTtmlToSrt(raw)
elif format.lower() == 'dcsubtitle':
srt = SubtitleHelper.__ConvertDCSubtitleToSrt(raw)
elif format.lower() == 'json':
srt = SubtitleHelper.__ConvertJsonSubtitleToSrt(raw)
elif format.lower() == 'm3u8srt':
srt = SubtitleHelper.__ConvertM3u8SrtToSubtitleToSrt(raw, url, proxy)
else:
error = "Uknown subtitle format: %s" % (format,)
raise NotImplementedError(error)
if replace:
Logger.Debug("Replacing SRT data: %s", replace)
for needle in replace:
srt = srt.replace(needle, replace[needle])
f = open(localCompletePath, 'w')
f.write(srt)
f.close()
Logger.Info("Saved SRT as %s", localCompletePath)
return localCompletePath
except:
Logger.Error("Error handling Subtitle file: [%s]", srt, exc_info=True)
return ""
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.Open(item.url, proxy=self.proxy)
json = JsonHelper(data, Logger.Instance())
videoData = json.GetValue("video")
if videoData:
part = item.CreateNewEmptyMediaPart()
spoofIp = self._GetSetting("spoof_ip", valueForNone="0.0.0.0")
if spoofIp:
part.HttpHeaders["X-Forwarded-For"] = spoofIp
# get the videos
videoUrls = videoData.get("videoReferences")
for videoUrl in videoUrls:
# Logger.Trace(videoUrl)
streamInfo = videoUrl['url']
if "manifest.f4m" in streamInfo:
continue
elif "master.m3u8" in streamInfo:
for s, b in M3u8.GetStreamsFromM3u8(
streamInfo, self.proxy, headers=part.HttpHeaders):
item.complete = True
part.AppendMediaStream(s, b)
#m3u8Data = UriHandler.Open(streamInfo, proxy=self.proxy)
#urls = Regexer.DoRegex(self.mediaUrlRegex, m3u8Data)
#Logger.Trace(urls)
#for url in urls:
#part.AppendMediaStream(url[1].strip(), url[0])
# subtitles
subtitles = videoData.get("subtitleReferences")
if subtitles:
Logger.Trace(subtitles)
subUrl = subtitles[0]["url"]
fileName = "%s.srt" % (EncodingHelper.EncodeMD5(subUrl), )
subData = UriHandler.Open(subUrl, proxy=self.proxy)
# correct the subs
regex = re.compile("^1(\d:)", re.MULTILINE)
subData = re.sub(regex, "0\g<1>", subData)
subData = re.sub("--> 1(\d):", "--> 0\g<1>:", subData)
localCompletePath = os.path.join(Config.cacheDir, fileName)
Logger.Debug("Saving subtitle to: %s", localCompletePath)
f = open(localCompletePath, 'w')
f.write(subData)
f.close()
part.Subtitle = localCompletePath
item.complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url, proxy=self.proxy)
item.MediaItemParts = []
part = item.create_new_empty_media_part()
base_encode = Regexer.do_regex('data-files="([^"]+)', data)
if base_encode:
Logger.debug("Loading video from BASE64 encoded JSON data")
base_encode = base_encode[-1]
json_data = EncodingHelper.decode_base64(base_encode)
json = JsonHelper(json_data, logger=Logger.instance())
Logger.trace(json)
# "flv": "http://media.dumpert.nl/flv/e2a926ff_10307954_804223649588516_151552487_n.mp4.flv",
# "tablet": "http://media.dumpert.nl/tablet/e2a926ff_10307954_804223649588516_151552487_n.mp4.mp4",
# "mobile": "http://media.dumpert.nl/mobile/e2a926ff_10307954_804223649588516_151552487_n.mp4.mp4",
streams = json.get_value()
for key in streams:
if key == "flv":
part.append_media_stream(streams[key], 1000)
elif key == "720p":
part.append_media_stream(streams[key], 1200)
elif key == "1080p":
part.append_media_stream(streams[key], 1600)
elif key == "tablet":
part.append_media_stream(streams[key], 800)
elif key == "mobile":
part.append_media_stream(streams[key], 450)
elif key == "embed" and streams[key].startswith("youtube"):
embed_type, youtube_id = streams[key].split(":")
url = "https://www.youtube.com/watch?v=%s" % (youtube_id, )
for s, b in YouTube.get_streams_from_you_tube(url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
else:
Logger.debug("Key '%s' was not used", key)
item.complete = True
Logger.trace("VideoItem updated: %s", item)
return item
youtube_id = Regexer.do_regex("class='yt-iframe'[^>]+src='https://www.youtube.com/embed/([^?]+)", data)
if youtube_id:
youtube_id = youtube_id[-1]
url = "https://www.youtube.com/watch?v=%s" % (youtube_id,)
for s, b in YouTube.get_streams_from_you_tube(url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item