def create_single_video_item(self, result_set):
""" Creates a MediaItem of type 'video' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
If the item is completely processed an no further data needs to be fetched
the self.complete property should be set to True. If not set to True, the
self.update_video_item method is called if the item is focussed or selected
for playback.
:param str result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'video' or 'audio' (despite the method's name).
:rtype: MediaItem|None
"""
if self.__hasAlreadyVideoItems:
# we already have items, so don't show this one, it will be a duplicate
return None
result_set = result_set.replace('\\x27', "'")
json_data = JsonHelper(result_set)
url = self.parentItem.url
title = json_data.get_value("name")
description = HtmlHelper.to_text(json_data.get_value("description"))
item = MediaItem(title, url, type="video")
item.description = description
item.thumb = self.parentItem.thumb
item.fanart = self.parentItem.fanart
return item
def update_video_api_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting UpdateChannelItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url, proxy=self.proxy)
json = JsonHelper(data, logger=Logger.instance())
videos = json.get_value("videoReferences")
subtitles = json.get_value("subtitleReferences")
Logger.trace(videos)
return self.__update_item_from_video_references(
item, videos, subtitles)
def extract_page_data(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
json = JsonHelper(data)
data = json.get_value("data")
Logger.trace(data)
if json.get_value("loadMore", fallback=False):
url, page = self.parentItem.url.rsplit("/", 1)
url = "{0}/{1}".format(url, int(page) + 1)
page_item = MediaItem("{0}".format(int(page) + 2), url)
page_item.type = "page"
items.append(page_item)
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
if not item.url.endswith(".js"):
data = UriHandler.open(item.url, proxy=self.proxy)
data_id = Regexer.do_regex(r'data-id="(\d+)"[^>]+data-playout', data)
if data_id is None:
Logger.warning("Cannot find stream-id for L1 stream.")
return item
data_url = "https://l1.bbvms.com/p/video/c/{}.json".format(data_id[0])
else:
data_url = item.url
data = UriHandler.open(data_url, proxy=self.proxy)
json = JsonHelper(data, logger=Logger.instance())
Logger.trace(json)
base_url = json.get_value("publicationData", "defaultMediaAssetPath")
streams = json.get_value("clipData", "assets")
item.MediaItemParts = []
part = item.create_new_empty_media_part()
for stream in streams:
url = stream.get("src", None)
if "://" not in url:
url = "{}{}".format(base_url, url)
bitrate = stream.get("bandwidth", None)
if url:
part.append_media_stream(url, bitrate)
if not item.thumb and json.get_value("thumbnails"):
url = json.get_value("thumbnails")[0].get("src", None)
if url and "http:/" not in url:
url = "%s%s" % (self.baseUrl, url)
item.thumb = url
item.complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
meta_data = UriHandler.open(item.url,
proxy=self.proxy,
referer=self.baseUrl)
meta = JsonHelper(meta_data)
stream_parts = meta.get_value("feed", "items")
for stream_part in stream_parts:
stream_url = stream_part["group"]["content"]
stream_url = stream_url.replace("{device}", "html5")
stream_url = "%s&format=json" % (stream_url, )
stream_data = UriHandler.open(stream_url, proxy=self.proxy)
stream = JsonHelper(stream_data)
# subUrls = stream.get_value("package", "video", "item", 0, "transcript", 0, "typographic") # NOSONAR
part = item.create_new_empty_media_part()
# m3u8Url = stream.get_value("package", "video", "item", 0, "rendition", 0, "src") # NOSONAR
# for s, b in M3u8.get_streams_from_m3u8(m3u8Url, self.proxy):
# item.complete = True
# part.append_media_stream(s, b)
rtmp_datas = stream.get_value("package", "video", "item", 0,
"rendition")
for rtmp_data in rtmp_datas:
rtmp_url = rtmp_data["src"]
rtmp_url = rtmp_url.replace("rtmpe://", "rtmp://")
bitrate = rtmp_data["bitrate"]
part.append_media_stream(rtmp_url, bitrate)
item.complete = True
Logger.trace("Media url: %s", item)
return item
def __extract_session_data(self, logon_data):
logon_json = JsonHelper(logon_data)
result_code = logon_json.get_value("statusCode")
if result_code != 200:
Logger.error("Error loging in: %s - %s",
logon_json.get_value("errorMessage"),
logon_json.get_value("errorDetails"))
return None, None, None
return logon_json.get_value("UID"), \
logon_json.get_value("UIDSignature"), \
logon_json.get_value("signatureTimestamp")
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
json = JsonHelper(data)
part = item.create_new_empty_media_part()
part.Subtitle = NpoStream.get_subtitle(json.get_value("mid"),
proxy=self.proxy)
for stream in json.get_value("videoStreams"):
if not stream["url"].startswith("odi"):
part.append_media_stream(stream["url"],
stream["bitrate"] / 1000)
item.complete = True
if item.has_media_item_parts():
return item
for s, b in NpoStream.get_streams_from_npo(None,
json.get_value("mid"),
proxy=self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def authenticate(self, username, password):
# Step 1: First initiate an authentication request
auth_request = self.__get_authentication_request(username)
auth_data = JsonHelper.dump(auth_request)
auth_headers = {
"X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
"Accept-Encoding": "identity",
"Content-Type": "application/x-amz-json-1.1"
}
auth_response = UriHandler.open(self.url,
proxy=self.__proxy,
params=auth_data,
additional_headers=auth_headers)
auth_response_json = JsonHelper(auth_response)
challenge_parameters = auth_response_json.get_value(
"ChallengeParameters")
if self.__logger:
self.__logger.trace(challenge_parameters)
challenge_name = auth_response_json.get_value("ChallengeName")
if not challenge_name == "PASSWORD_VERIFIER":
if self.__logger:
self.__logger.error("Cannot start authentication challenge")
return None
# Step 2: Respond to the Challenge with a valid ChallengeResponse
challenge_request = self.__get_challenge_response_request(
challenge_parameters, password)
challenge_data = JsonHelper.dump(challenge_request)
challenge_headers = {
"X-Amz-Target":
"AWSCognitoIdentityProviderService.RespondToAuthChallenge",
"Content-Type": "application/x-amz-json-1.1"
}
auth_response = UriHandler.open(self.url,
proxy=self.__proxy,
params=challenge_data,
additional_headers=challenge_headers)
auth_response_json = JsonHelper(auth_response)
if "message" in auth_response_json.json:
self.__logger.error("Error logging in: %s",
auth_response_json.get_value("message"))
return None, None
id_token = auth_response_json.get_value("AuthenticationResult",
"IdToken")
refresh_token = auth_response_json.get_value("AuthenticationResult",
"RefreshToken")
return id_token, refresh_token
def __get_online_version_from_github(self, include_alpha_beta=False):
""" Retrieves the current online version.
:param bool include_alpha_beta: should we include alpha/beta releases?
:return: Returns the current online version or `None` of no version was found.
:rtype: None|Version
"""
data = self.__uriHandler.open(self.updateUrl, no_cache=True)
json_data = JsonHelper(data)
version_tag = "tag_name"
online_versions = [
r[version_tag].lstrip("v").replace("-", "~")
for r in json_data.get_value() if bool(r[version_tag]) and (
not r["prerelease"] or include_alpha_beta)
]
if not bool(online_versions):
return None
max_version = None
for online_version_data in online_versions:
online_version = Version(online_version_data)
if not include_alpha_beta and online_version.buildType is not None:
self.__logger.trace("Ignoring %s", online_version)
continue
self.__logger.trace("Found possible version: %s", online_version)
if online_version > max_version:
max_version = online_version
return max_version
def __get_online_version_from_bitbucket(self, include_alpha_beta=False):
""" Retrieves the current online version.
:param bool include_alpha_beta: should we include alpha/beta releases?
:return: Returns the current online version or `None` of no version was found.
:rtype: None|Version
"""
data = self.__uriHandler.open(self.updateUrl, no_cache=True)
json_data = JsonHelper(data)
online_downloads = [d for d in json_data.get_value("values") if self.__is_valid_update(d)]
if len(online_downloads) == 0:
return None
max_version = None
for online_download in online_downloads:
online_parts = online_download['name'].rsplit(".", 1)[0].split("-")
if len(online_parts) < 2:
continue
# fix the problem that a ~ is preventing downloads on BitBucket
online_version_data = online_parts[1].replace("alpha", "~alpha").replace("beta", "~beta")
online_version = Version(online_version_data)
if not include_alpha_beta and online_version.buildType is not None:
self.__logger.trace("Ignoring %s", online_version)
continue
self.__logger.trace("Found possible version: %s", online_version)
if online_version > max_version:
max_version = online_version
return max_version
def update_json_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
headers = {}
if self.localIP:
headers.update(self.localIP)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=headers)
video_data = JsonHelper(data)
stream_data = video_data.get_value("mediaAssetsOnDemand")
if not stream_data:
return item
use_adaptive = AddonSettings.use_adaptive_stream_add_on()
stream_data = stream_data[0]
part = item.create_new_empty_media_part()
if "hlsUrl" in stream_data:
hls_url = stream_data["hlsUrl"]
if use_adaptive:
stream = part.append_media_stream(hls_url, 0)
M3u8.set_input_stream_addon_input(stream,
self.proxy,
headers=headers)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(hls_url,
self.proxy,
headers=headers):
item.complete = True
part.append_media_stream(s, b)
if "timedTextSubtitlesUrl" in stream_data and stream_data[
"timedTextSubtitlesUrl"]:
sub_url = stream_data["timedTextSubtitlesUrl"].replace(
".ttml", ".vtt")
sub_url = HtmlEntityHelper.url_decode(sub_url)
part.Subtitle = SubtitleHelper.download_subtitle(sub_url,
format="webvtt")
return item
def list_some_videos(self, data):
""" If there was a Lastest section in the data return those video files
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
if not self.__showSomeVideosInListing:
return data, items
json_data = JsonHelper(data)
sections = json_data.get_value("relatedVideoContent", "relatedVideosAccordion")
sections = list(filter(lambda s: s['type'] not in self.__excludedTabs, sections))
Logger.debug("Found %s folders/tabs", len(sections))
if len(sections) == 1:
# we should exclude that tab from the folders list and show the videos here
self.__listedRelatedTab = sections[0]["type"]
# otherwise the default "RELATED_VIDEO_TABS_LATEST" is used
Logger.debug("Excluded tab '%s' which will be show as videos", self.__listedRelatedTab)
for section in sections:
if not section["type"] == self.__listedRelatedTab:
continue
for video_data in section['videos']:
items.append(self.create_json_item(video_data))
return data, items
def load_channel_data(self, data):
""" Adds the channel items to the listing.
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
now = datetime.datetime.now()
try:
server_time = UriHandler.open("https://www.svtplay.se/api/server_time",
proxy=self.proxy, no_cache=True)
server_time_json = JsonHelper(server_time)
server_time = server_time_json.get_value("time")
except:
Logger.error("Error determining server time", exc_info=True)
server_time = "%04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
data = UriHandler.open(
"https://www.svtplay.se/api/channel_page?now=%s" % (server_time, ),
proxy=self.proxy)
return data, items
def extract_slug_data(self, data):
""" Extracts the correct Slugged Data for tabbed items
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Extracting Slugged data during pre-processing")
data, items = self.extract_json_data(data)
json = JsonHelper(data)
slugs = json.get_value("relatedVideoContent", "relatedVideosAccordion")
for slug_data in slugs:
tab_slug = "?tab=%s" % (slug_data["slug"], )
if not self.parentItem.url.endswith(tab_slug):
continue
for item in slug_data["videos"]:
i = self.create_json_item(item)
if i:
items.append(i)
return data, items
def extract_main_list_json(self, data):
""" Extracts the main list JSON data from the HTML response.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
data, items = self.add_others(data)
start_string = "window.__REDUX_STATE__ = "
start_data = data.index(start_string)
end_data = data.index("</script><script async=")
data = data[start_data + len(start_string):end_data]
data = JsonHelper(data)
letters = data.get_value("reduxAsyncConnect", "page", "components", 1,
"data", "items", 1, "data", "items")
for letter_data in letters:
letter_data = letter_data["data"]
Logger.trace("Processing '%s'", letter_data["title"])
for item in letter_data["items"]:
episode = self.create_json_episode_item(item)
items.append(episode)
return data, items
def pre_process_folder_list(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
if "episode.json" in self.parentItem.url:
Logger.debug("Fetching Carousel data")
json = JsonHelper(data)
data = json.get_value("carousel")
Logger.debug("Pre-Processing finished")
return data, items
def extract_day_items(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
json = JsonHelper(data)
page_items = json.get_value('items')
for item in page_items:
video_item = self.create_json_video_item(item, prepend_serie=True)
if video_item:
items.append(video_item)
return data, items
def renew_token(self, refresh_token):
"""
Sets a new access token on the User using the refresh token. The basic expire time of the
refresh token is 30 days:
http://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-with-identity-providers.html
:param str refresh_token: Token to use for refreshing the authorization token.
"""
refresh_request = {
"AuthParameters": {
"REFRESH_TOKEN": refresh_token
},
"ClientId": self.client_id,
"AuthFlow": "REFRESH_TOKEN"
}
refresh_headers = {
"X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
"Content-Type": "application/x-amz-json-1.1"
}
refresh_request_data = JsonHelper.dump(refresh_request)
refresh_response = UriHandler.open(self.url,
proxy=self.__proxy,
params=refresh_request_data,
additional_headers=refresh_headers)
refresh_json = JsonHelper(refresh_response)
id_token = refresh_json.get_value("AuthenticationResult", "IdToken")
return id_token
def update_json_video(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item: %s', item.name)
data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=self.httpHeaders)
json_data = JsonHelper(data)
streams = json_data.get_value("formats")
if not streams:
return item
qualities = {"720p": 1600, "480p": 1200, "360p": 500, "other": 0} # , "http-hls": 1500, "3gp-mob01": 300, "flv-web01": 500}
part = item.create_new_empty_media_part()
urls = []
for stream in streams:
url = stream["url"].values()[-1]
if url in urls:
# duplicate url, ignore
continue
urls.append(url)
# actually process the url
if not url.endswith(".m3u8"):
part.append_media_stream(
url=url,
bitrate=qualities.get(stream.get("name", "other"), 0)
)
item.complete = True
# elif AddonSettings.use_adaptive_stream_add_on():
# content_type, url = UriHandler.header(url, self.proxy)
# stream = part.append_media_stream(url, 0)
# M3u8.SetInputStreamAddonInput(stream, self.proxy)
# item.complete = True
else:
content_type, url = UriHandler.header(url, self.proxy)
for s, b in M3u8.get_streams_from_m3u8(url, self.proxy):
item.complete = True
part.append_media_stream(s, b)
return item
def update_video_item(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv
# http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url, proxy=self.proxy)
part = item.create_new_empty_media_part()
if "mediazone.vrt.be" not in item.url:
# Extract actual media data
video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0]
url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, )
data = UriHandler.open(url, proxy=self.proxy)
json = JsonHelper(data)
urls = json.get_value("targetUrls")
for url_info in urls:
Logger.trace(url_info)
if url_info["type"].lower() != "hls":
continue
hls_url = url_info["url"]
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
return item
def update_video_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
# sources
part = item.create_new_empty_media_part()
# high, web, mobile, url
media_sources = media_info.json.get("sources", {})
for quality in media_sources:
url = media_sources[quality]
if quality == "high":
bitrate = 2000
elif quality == "web":
bitrate = 800
elif quality == "mobile":
bitrate = 400
else:
bitrate = 0
part.append_media_stream(url, bitrate)
# geoLocRestriction
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
item.complete = True
return item
def make_episode_dictionary_array(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_data = JsonHelper(data)
dict_items = json_data.get_value("items", fallback=[])
for item in dict_items:
if item == "banners" or item == "curators":
continue
items.append(self.create_episode_item(dict_items[item]))
Logger.debug("Pre-Processing finished")
data = ""
return data, items
def update_music_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_music_item for %s (%s)', item.name, self.channelName)
url, data = item.url.split("?")
data = UriHandler.open(url, proxy=self.proxy, params=data, additional_headers=item.HttpHeaders)
Logger.trace(data)
json_data = JsonHelper(data)
url = json_data.get_value("url", fallback=None)
if url:
item.append_single_stream(url)
item.Complete = True
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
if not item.url.endswith("m3u8"):
data = UriHandler.open(item.url, proxy=self.proxy)
json_data = Regexer.do_regex(self.mediaUrlRegex, data)
if not json_data:
Logger.error("Cannot find JSON stream info.")
return item
json = JsonHelper(json_data[0])
Logger.trace(json.json)
stream = json.get_value("source", "hls")
if stream is None:
stream = json.get_value("mzsource", "hls")
Logger.debug("Found HLS: %s", stream)
else:
stream = item.url
part = item.create_new_empty_media_part()
for s, b in M3u8.get_streams_from_m3u8(stream, self.proxy):
item.complete = True
part.append_media_stream(s, b)
# var playerConfig = {"id":"mediaplayer","width":"100%","height":"100%","autostart":"false","image":"http:\/\/www.ketnet.be\/sites\/default\/files\/thumb_5667ea22632bc.jpg","brand":"ketnet","source":{"hls":"http:\/\/vod.stream.vrt.be\/ketnet\/_definst_\/mp4:ketnet\/2015\/12\/Ben_ik_familie_van_R001_A0023_20151208_143112_864.mp4\/playlist.m3u8"},"analytics":{"type_stream":"vod","playlist":"Ben ik familie van?","program":"Ben ik familie van?","episode":"Ben ik familie van?: Warre - Aflevering 3","parts":"1","whatson":"270157835527"},"title":"Ben ik familie van?: Warre - Aflevering 3","description":"Ben ik familie van?: Warre - Aflevering 3"}
return item
def update_video_item_javascript(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
url_parts = item.url.rsplit("/", 3)
if url_parts[-3] == "aflevering":
video_id = url_parts[-2]
else:
video_id = url_parts[-1]
Logger.debug("Found videoId '%s' for '%s'", video_id, item.url)
url = "https://omroepzeeland.bbvms.com/p/regiogrid/q/sourceid_string:{}*.js".format(
video_id)
data = UriHandler.open(url, proxy=self.proxy)
json_data = Regexer.do_regex(r'var opts\s*=\s*({.+});\W*//window',
data)
Logger.debug("Found jsondata with size: %s", len(json_data[0]))
json_data = JsonHelper(json_data[0])
clip_data = json_data.get_value("clipData", "assets")
server = json_data.get_value("publicationData",
"defaultMediaAssetPath")
part = item.create_new_empty_media_part()
for clip in clip_data:
part.append_media_stream("{}{}".format(server, clip["src"]),
int(clip["bandwidth"]))
item.complete = True
return item
def __send_haste_bin(self, code):
""" Sends a logfile to paste.kodi.tv
:param str code: The content to post
"""
response = UriHandler.open("https://paste.kodi.tv/documents", params=code.encode(), proxy=self.__proxy)
json = JsonHelper(response)
key = json.get_value("key")
if not key:
raise IOError(json.get_value("message"))
url = "https://paste.kodi.tv/{}".format(key)
if self.__logger:
self.__logger.info("HasteBin Url: %s", url)
return url
def add_missing_live_streams(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
slam = MediaItem(
"Slam! TV",
"https://hls.slam.nl/streaming/hls/SLAM!/playlist.m3u8")
slam.icon = self.icon
slam.thumb = self.noImage
slam.type = "video"
slam.isLive = True
items.append(slam)
slam_fm = MediaItem(
"Slam! FM", "https://18973.live.streamtheworld.com/SLAM_AAC.aac"
"?ttag=PLAYER%3ANOPREROLL&tdsdk=js-2.9"
"&pname=TDSdk&pversion=2.9&banners=none")
slam_fm.icon = self.icon
slam_fm.thumb = self.noImage
slam_fm.type = "audio"
slam_fm.isLive = True
slam_fm.append_single_stream(slam_fm.url)
slam_fm.complete = True
items.append(slam_fm)
data = JsonHelper(data)
for e in data.get_value("includes", "Entry"):
self.__liveData[e["sys"]["id"]] = e
for e in data.get_value("includes", "Asset"):
self.__liveData[e["sys"]["id"]] = e
return data, items
def send_files(self, name, file_paths):
""" Sends multiple files.
:param str|unicode name: Name for the gist/paste.
:param list[str|unicode] file_paths: List of file paths.
:return: The result of the upload.
:rtype: any
"""
if self.__mode != "gist":
raise ValueError("Invalid mode for multiple files")
params = {
"description": name,
"public": False,
"files": {
# name: {
# "content": code
# }
}
}
for file_path in file_paths:
if not os.path.isfile(file_path):
continue
code = self.__read_file_bytes(file_path)
file_name = os.path.split(file_path)
params["files"][file_name[-1]] = {"content": code}
headers = {
"Content-Type": "application/json"
}
post_data = JsonHelper.dump(params, pretty_print=False)
data = UriHandler.open("https://api.github.com/gists", params=post_data,
proxy=self.__proxy, additional_headers=headers)
if not data:
raise IOError("Error posting Gist to GitHub")
json_data = JsonHelper(data)
url = json_data.get_value("html_url")
if self.__logger:
self.__logger.info("Gist: %s", url)
# minify with google
# POST https://www.googleapis.com/urlshortener/v1/url
# Content-Type: application/json
shortener = {"longUrl": url}
google = "https://www.googleapis.com/urlshortener/v1/url?key=%s" % (self.__apiKey,)
google_data = UriHandler.open(google, params=JsonHelper.dump(shortener, False),
proxy=self.__proxy,
additional_headers={"Content-Type": "application/json"})
google_url = JsonHelper(google_data).get_value("id")
if self.__logger:
self.__logger.info("Goo.gl: %s", google_url)
return google_url
def update_live_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
part = item.create_new_empty_media_part()
hls_url = media_info.get_value("streamUrl")
if hls_url is not None and "m3u8" in hls_url:
Logger.debug("Found HLS url for %s: %s",
media_info.json["streamName"], hls_url)
for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy):
part.append_media_stream(s, b)
item.complete = True
else:
Logger.debug("No HLS url found for %s. Fetching RTMP Token.",
media_info.json["streamName"])
# fetch the token:
token_url = "%s/api/media/streaming?streamname=%s" \
% (self.baseUrl, media_info.json["streamName"])
token_data = UriHandler.open(token_url,
proxy=self.proxy,
additional_headers=item.HttpHeaders,
no_cache=True)
token_data = JsonHelper(token_data)
token = token_data.get_value("token")
Logger.debug("Found token '%s' for '%s'", token,
media_info.json["streamName"])
rtmp_url = "rtmp://rtmp.rtbf.be/livecast/%s?%s pageUrl=%s tcUrl=rtmp://rtmp.rtbf.be/livecast" \
% (media_info.json["streamName"], token, self.baseUrl)
rtmp_url = self.get_verifiable_video_url(rtmp_url)
part.append_media_stream(rtmp_url, 0)
item.complete = True
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
return item
def load_all_episodes(self, data):
episodes_json = []
data, items = self.add_live_channel(data)
for i in range(0, 20):
url = "https://at5news.vinsontv.com/api/news?source=web&slug=tv&page={}".format(
i)
data = UriHandler.open(url, proxy=self.proxy)
json_data = JsonHelper(data)
item_data = json_data.get_value("category", "news", fallback=[])
episodes_json += item_data
if len(item_data) < json_data.get_value("pageSize"):
break
dummy = JsonHelper("{}")
dummy.json = episodes_json
return dummy, items
