def extract_main_list_json(self, data):
""" Extracts the main list JSON data from the HTML response.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
data, items = self.add_others(data)
start_string = "window.__REDUX_STATE__ = "
start_data = data.index(start_string)
end_data = data.index("</script><script async=")
data = data[start_data + len(start_string):end_data]
data = JsonHelper(data)
letters = data.get_value("reduxAsyncConnect", "page", "components", 1,
"data", "items", 1, "data", "items")
for letter_data in letters:
letter_data = letter_data["data"]
Logger.trace("Processing '%s'", letter_data["title"])
for item in letter_data["items"]:
episode = self.create_json_episode_item(item)
items.append(episode)
return data, items
def update_video_item(self, item):
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
media_regex = 'data-media="([^"]+)"'
media_info = Regexer.do_regex(media_regex, data)[0]
media_info = HtmlEntityHelper.convert_html_entities(media_info)
media_info = JsonHelper(media_info)
Logger.trace(media_info)
# sources
part = item.create_new_empty_media_part()
# high, web, mobile, url
media_sources = media_info.json.get("sources", {})
for quality in media_sources:
url = media_sources[quality]
if quality == "high":
bitrate = 2000
elif quality == "web":
bitrate = 800
elif quality == "mobile":
bitrate = 400
else:
bitrate = 0
part.append_media_stream(url, bitrate)
# geoLocRestriction
item.isGeoLocked = not media_info.get_value(
"geoLocRestriction", fallback="world") == "world"
item.complete = True
return item
def __iterate_results(self,
url_format,
results_per_page=150,
max_iterations=10):
""" Retrieves the full dataset for a multi-set search action.
:param str url_format: The url format with start and count placeholders
:param int results_per_page: The maximum results per request
:param int max_iterations: The maximum number of iterations
:returns A Json response with all results
:rtype JsonHelper
Url format should be like:
https://urplay.se/api/bff/v1/search?product_type=series&rows={}&start={}
"""
results = None
for p in range(0, max_iterations):
url = url_format.format(results_per_page, p * results_per_page)
data = UriHandler.open(url)
json_data = JsonHelper(data)
result_items = json_data.get_value("results", fallback=[])
if results is None:
results = json_data
else:
results.json["results"] += result_items
if len(result_items) < results_per_page:
break
return results or ""
def extract_json_episodes(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
data = Regexer.do_regex(
r'window.__DATA__ = ([\w\W]+?});\s*window.__PUSH_STATE__', data)[0]
json_data = JsonHelper(data)
main_container = [
m for m in json_data.get_value("children")
if m["type"] == "MainContainer"
]
line_list = [
item for item in main_container[0]["children"]
if item["type"] == "LineList"
]
line_list = line_list[0]["props"]
json_data.json = line_list
return json_data, items
def __is_already_logged_on(self, username):
""" Check if the given user is logged on and sets what packages he/she has.
:param str username:
:return: Indicator if the user is alreadly logged in
:rtype: bool
"""
me = UriHandler.open("https://disco-api.dplay.se/users/me", proxy=self.proxy, no_cache=True)
if UriHandler.instance().status.code >= 300:
return False
account_data = JsonHelper(me)
signed_in_user = account_data.get_value("data", "attributes", "username")
if signed_in_user is not None and signed_in_user != username:
# Log out
UriHandler.open("https://disco-api.dplay.se/logout", data="", proxy=self.proxy, no_cache=True)
return False
logged_in = not account_data.get_value("data", "attributes", "anonymous")
if logged_in:
Logger.debug("Already logged in")
packages = account_data.get_value("data", "attributes", "packages", fallback=[])
self.__has_premium = "Premium" in packages
return True
else:
return False
def __get_api_url(self, operation, hash_value, variables=None):
""" Generates a GraphQL url
:param str operation: The operation to use
:param str hash_value: The hash of the Query
:param dict variables: Any variables to pass
:return: A GraphQL string
:rtype: str
"""
extensions = {
"persistedQuery": {
"version": 1,
"sha256Hash": hash_value
}
}
extensions = HtmlEntityHelper.url_encode(
JsonHelper.dump(extensions, pretty_print=False))
final_vars = {"order_by": "NAME", "per_page": 1000}
if variables:
final_vars = variables
final_vars = HtmlEntityHelper.url_encode(
JsonHelper.dump(final_vars, pretty_print=False))
url = "https://graphql.tv4play.se/graphql?" \
"operationName={}&" \
"variables={}&" \
"extensions={}".format(operation, final_vars, extensions)
return url
def __update_video_from_brightcove(self, item, data,
use_adaptive_with_encryption):
""" Updates an existing MediaItem with more data based on an MPD stream.
:param str data: Stream info retrieved from BrightCove.
:param bool use_adaptive_with_encryption: Do we use the Adaptive InputStream add-on?
:param MediaItem item: The original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
part = item.create_new_empty_media_part()
# Then try the new BrightCove JSON
bright_cove_regex = '<video[^>]+data-video-id="(?<videoId>[^"]+)[^>]+data-account="(?<videoAccount>[^"]+)'
bright_cove_data = Regexer.do_regex(
Regexer.from_expresso(bright_cove_regex), data)
if not bright_cove_data:
Logger.warning("Error updating using BrightCove data: %s", item)
return item
Logger.info("Found new BrightCove JSON data")
bright_cove_url = 'https://edge.api.brightcove.com/playback/v1/accounts/' \
'%(videoAccount)s/videos/%(videoId)s' % bright_cove_data[0]
headers = {
"Accept":
"application/json;pk=BCpkADawqM3ve1c3k3HcmzaxBvD8lXCl89K7XEHiKutxZArg2c5RhwJHJANOwPwS_4o7UsC4RhIzXG8Y69mrwKCPlRkIxNgPQVY9qG78SJ1TJop4JoDDcgdsNrg"
}
bright_cove_data = UriHandler.open(bright_cove_url,
additional_headers=headers)
bright_cove_json = JsonHelper(bright_cove_data)
streams = [
d for d in bright_cove_json.get_value("sources")
if d["container"] == "M2TS"
]
# Old filter
# streams = filter(lambda d: d["container"] == "M2TS", bright_cove_json.get_value("sources"))
if not streams:
Logger.warning("Error extracting streams from BrightCove data: %s",
item)
return item
# noinspection PyTypeChecker
stream_url = streams[0]["src"]
# these streams work better with the the InputStreamAddon because it removes the
# "range" http header
if use_adaptive_with_encryption:
Logger.info("Using InputStreamAddon for playback of HLS stream")
strm = part.append_media_stream(stream_url, 0)
M3u8.set_input_stream_addon_input(strm)
item.complete = True
return item
for s, b in M3u8.get_streams_from_m3u8(stream_url):
item.complete = True
part.append_media_stream(s, b)
return item
def update_video_item(self, item):
"""
Accepts an item. It returns an updated item. Usually retrieves the MediaURL
and the Thumb! It should return a completed item.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv
# http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf
# now the mediaurl is derived. First we try WMV
data = UriHandler.open(item.url)
part = item.create_new_empty_media_part()
if "mediazone.vrt.be" not in item.url:
# Extract actual media data
video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0]
url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, )
data = UriHandler.open(url)
json = JsonHelper(data)
urls = json.get_value("targetUrls")
for url_info in urls:
Logger.trace(url_info)
if url_info["type"].lower() != "hls":
continue
hls_url = url_info["url"]
for s, b in M3u8.get_streams_from_m3u8(hls_url):
part.append_media_stream(s, b)
item.complete = True
return item
def __get_api_persisted_url(self, operation, hash_value,
variables): # NOSONAR
""" Generates a GraphQL url
:param str operation: The operation to use
:param str hash_value: The hash of the Query
:param dict variables: Any variables to pass
:return: A GraphQL string
:rtype: str
"""
extensions = {
"persistedQuery": {
"version": 1,
"sha256Hash": hash_value
}
}
extensions = HtmlEntityHelper.url_encode(
JsonHelper.dump(extensions, pretty_print=False))
variables = HtmlEntityHelper.url_encode(
JsonHelper.dump(variables, pretty_print=False))
url = "https://graph.kijk.nl/graphql?" \
"operationName={}&" \
"variables={}&" \
"extensions={}".format(operation, variables, extensions)
return url
def __update_video(self, item, data):
if not item.url.startswith("https://api.viervijfzes.be/content/"):
regex = 'data-video-*id="([^"]+)'
m3u8_url = Regexer.do_regex(regex, data)[-1]
# we either have an URL now or an uuid
else:
m3u8_url = item.url.rsplit("/", 1)[-1]
if ".m3u8" not in m3u8_url:
Logger.info("Not a direct M3u8 file. Need to log in")
url = "https://api.viervijfzes.be/content/%s" % (m3u8_url, )
# We need to log in
if not self.loggedOn:
self.log_on()
# add authorization header
authentication_header = {
"authorization": self.__idToken,
"content-type": "application/json"
}
data = UriHandler.open(url, additional_headers=authentication_header)
json_data = JsonHelper(data)
m3u8_url = json_data.get_value("video", "S")
# Geo Locked?
if "/geo/" in m3u8_url.lower():
# set it for the error statistics
item.isGeoLocked = True
part = item.create_new_empty_media_part()
item.complete = M3u8.update_part_with_m3u8_streams(
part, m3u8_url, channel=self, encrypted=False)
return item
def extract_day_items(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
json = JsonHelper(data)
page_items = json.get_value('items')
for item in page_items:
video_item = self.create_json_video_item(item, prepend_serie=True)
if video_item:
items.append(video_item)
return data, items
def pre_process_folder_list(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
if "episode.json" in self.parentItem.url:
Logger.debug("Fetching Carousel data")
json = JsonHelper(data)
data = json.get_value("carousel")
Logger.debug("Pre-Processing finished")
return data, items
def make_episode_dictionary_array(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_data = JsonHelper(data)
dict_items = json_data.get_value("items", fallback=[])
for item in dict_items:
if item == "banners" or item == "curators":
continue
items.append(self.create_episode_item(dict_items[item]))
Logger.debug("Pre-Processing finished")
data = ""
return data, items
def extract_page_data(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
items = []
json = JsonHelper(data)
data = json.get_value("data")
Logger.trace(data)
if json.get_value("loadMore", fallback=False):
url, page = self.parentItem.url.rsplit("/", 1)
url = "{0}/{1}".format(url, int(page) + 1)
page_item = MediaItem("{0}".format(int(page) + 2), url)
page_item.type = "page"
items.append(page_item)
return data, items
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName)
# Get the MZID
data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders)
json_data = Regexer.do_regex(r'<script type="application/ld\+json">(.*?)</script>', data)
json_info = JsonHelper(json_data[-1])
video_id = json_info.get_value("video", "@id")
publication_id = json_info.get_value("publication", -1, "@id")
mzid = "{}${}".format(publication_id, video_id)
return self.update_video_for_mzid(item, mzid)
def update_music_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_music_item for %s (%s)', item.name, self.channelName)
url, data = item.url.split("?")
data = UriHandler.open(url, params=data, additional_headers=item.HttpHeaders)
Logger.trace(data)
json_data = JsonHelper(data)
url = json_data.get_value("url", fallback=None)
if url:
item.append_single_stream(url)
item.Complete = True
return item
def renew_token(self, refresh_token):
"""
Sets a new access token on the User using the refresh token. The basic expire time of the
refresh token is 30 days:
http://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-with-identity-providers.html
:param str refresh_token: Token to use for refreshing the authorization token.
"""
refresh_request = {
"AuthParameters": {
"REFRESH_TOKEN": refresh_token
},
"ClientId": self.client_id,
"AuthFlow": "REFRESH_TOKEN"
}
refresh_headers = {
"X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
"Content-Type": "application/x-amz-json-1.1"
}
refresh_request_data = JsonHelper.dump(refresh_request)
refresh_response = UriHandler.open(self.url, proxy=self.__proxy,
params=refresh_request_data,
additional_headers=refresh_headers)
refresh_json = JsonHelper(refresh_response)
id_token = refresh_json.get_value("AuthenticationResult", "IdToken")
return id_token
def create_single_video_item(self, result_set):
""" Creates a MediaItem of type 'video' using the result_set from the regex.
This method creates a new MediaItem from the Regular Expression or Json
results <result_set>. The method should be implemented by derived classes
and are specific to the channel.
If the item is completely processed an no further data needs to be fetched
the self.complete property should be set to True. If not set to True, the
self.update_video_item method is called if the item is focussed or selected
for playback.
:param str result_set: The result_set of the self.episodeItemRegex
:return: A new MediaItem of type 'video' or 'audio' (despite the method's name).
:rtype: MediaItem|None
"""
if self.__hasAlreadyVideoItems:
# we already have items, so don't show this one, it will be a duplicate
return None
result_set = result_set.replace('\\x27', "'")
json_data = JsonHelper(result_set)
url = self.parentItem.url
title = json_data.get_value("name")
description = HtmlHelper.to_text(json_data.get_value("description"))
item = MediaItem(title, url, type="video")
item.description = description
return item
def update_live_channel(self, item):
""" Updates an existing live stream MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url, no_cache=True)
manifest = JsonHelper(data)
if "nonPlayable" in manifest.json and manifest.json["nonPlayable"]:
Logger.error("Cannot update Live: %s", item)
return item
source = manifest.get_value("sourceMedium")
if source == "audio":
return self.__update_live_audio(item, manifest)
else:
return self.__update_live_video(item, manifest)
def extract_json(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
json_data = Regexer.do_regex('type="application/json">([^<]+)<', data)
if not json_data:
Logger.warning("No JSON data found.")
return data, items
json = JsonHelper(json_data[0])
result = []
for key, value in json.json.items():
result.append(value)
value["title"] = key
# set new json and return JsonHelper object
json.json = result
return json, items
def update_json_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
headers = {}
if self.localIP:
headers.update(self.localIP)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=headers)
video_data = JsonHelper(data)
stream_data = video_data.get_value("mediaAssetsOnDemand")
if not stream_data:
return item
use_adaptive = AddonSettings.use_adaptive_stream_add_on()
stream_data = stream_data[0]
part = item.create_new_empty_media_part()
if "hlsUrl" in stream_data:
hls_url = stream_data["hlsUrl"]
if use_adaptive:
stream = part.append_media_stream(hls_url, 0)
M3u8.set_input_stream_addon_input(stream,
self.proxy,
headers=headers)
item.complete = True
else:
for s, b in M3u8.get_streams_from_m3u8(hls_url,
self.proxy,
headers=headers):
item.complete = True
part.append_media_stream(s, b)
if "timedTextSubtitlesUrl" in stream_data and stream_data[
"timedTextSubtitlesUrl"]:
sub_url = stream_data["timedTextSubtitlesUrl"].replace(
".ttml", ".vtt")
sub_url = HtmlEntityHelper.url_decode(sub_url)
part.Subtitle = SubtitleHelper.download_subtitle(sub_url,
format="webvtt")
return item
def __get_online_version_from_github(self, include_alpha_beta=False):
""" Retrieves the current online version.
:param bool include_alpha_beta: should we include alpha/beta releases?
:return: Returns the current online version or `None` of no version was found.
:rtype: None|Version
"""
data = self.__uriHandler.open(self.updateUrl, no_cache=True)
json_data = JsonHelper(data)
version_tag = "tag_name"
online_versions = [
r[version_tag].lstrip("v").replace("-", "~")
for r in json_data.get_value() if bool(r[version_tag]) and (
not r["prerelease"] or include_alpha_beta)
]
if not bool(online_versions):
return None
max_version = None
for online_version_data in online_versions:
online_version = Version(online_version_data)
if not include_alpha_beta and online_version.buildType is not None:
self.__logger.trace("Ignoring %s", online_version)
continue
self.__logger.trace("Found possible version: %s", online_version)
if online_version > max_version:
max_version = online_version
return max_version
def update_video_api_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting UpdateChannelItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url, proxy=self.proxy)
json = JsonHelper(data, logger=Logger.instance())
videos = json.get_value("videoReferences")
subtitles = json.get_value("subtitleReferences")
Logger.trace(videos)
return self.__update_item_from_video_references(
item, videos, subtitles)
def update_live_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
if "icecast" in item.url:
item.type = "audio"
data = UriHandler.open(item.url)
stream_info = JsonHelper(data)
url = stream_info.get_value("data")
item.url = url
return self.update_video_item(item)
def add_categories(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
cat = MediaItem(
"\b.: Categorieën :.",
"http://m.schooltv.nl/api/v1/categorieen.json?size=100")
cat.complete = True
cat.dontGroup = True
items.append(cat)
tips = MediaItem(
"\b.: Tips :.",
"http://m.schooltv.nl/api/v1/programmas/tips.json?size=100")
tips.complete = True
tips.dontGroup = True
items.append(tips)
data = JsonHelper(data)
ages = MediaItem("\b.: Leeftijden :.", "")
ages.complete = True
ages.dontGroup = True
for age in ("0-4", "5-6", "7-8", "9-12", "13-15", "16-18"):
age_item = MediaItem(
"%s Jaar" % (age, ),
"http://m.schooltv.nl/api/v1/leeftijdscategorieen/%s/afleveringen.json?"
"size=%s&sort=Nieuwste" % (age, self.__PageSize))
age_item.complete = True
age_item.dontGroup = True
ages.items.append(age_item)
# We should list programs instead of videos, so just prefill them here.
for program in data.get_value():
if age in program['ageGroups']:
age_item.items.append(self.create_episode_item(program))
items.append(ages)
Logger.debug("Pre-Processing finished")
return data, items
def update_json_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
data = UriHandler.open(item.url)
video_data = JsonHelper(data)
stream_data = video_data.get_value("playable")
if not stream_data:
return item
part = item.create_new_empty_media_part()
for stream_info in stream_data["assets"]:
url = stream_info["url"]
stream_type = stream_info["format"]
if stream_type == "HLS":
item.complete = M3u8.update_part_with_m3u8_streams(part, url)
else:
Logger.warning("Found unknow stream type: %s", stream_type)
if "subtitles" not in stream_data or not stream_data["subtitles"]:
return item
for sub in stream_data["subtitles"]:
sub_url = None
sub_type = sub["type"]
default_sub = sub["defaultOn"]
if default_sub:
sub_url = sub["webVtt"]
sub_type = "webvtt" # set Retrospect type
if sub_url:
part.Subtitle = SubtitleHelper.download_subtitle(
sub_url, format=sub_type)
break
return item
def update_video_item(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url, proxy=self.proxy)
# get the playlist GUID
playlist_guids = Regexer.do_regex(
"<div[^>]+data-playlist-id='([^']+)'[^>]+></div>", data)
if not playlist_guids:
# let's try the alternative then (for the new channels)
playlist_guids = Regexer.do_regex(
'local_playlist[", -]+([a-f0-9]{20})"', data)
playlist_guid = playlist_guids[0]
play_list_url = "http://api.mtvnn.com/v2/nl/NL/local_playlists/{}.json?video_format=m3u8".format(
playlist_guid)
data = UriHandler.open(play_list_url, proxy=self.proxy)
from resources.lib.helpers.jsonhelper import JsonHelper
from resources.lib.streams.m3u8 import M3u8
json_data = JsonHelper(data)
m3u8_url = json_data.get_value("local_playlist_videos", 0, "url")
part = item.create_new_empty_media_part()
item.complete = M3u8.update_part_with_m3u8_streams(part,
m3u8_url,
proxy=self.proxy,
channel=self,
encrypted=True)
return item
def update_video_item(self, item):
""" Updates an existing MediaItem with more data.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
:param MediaItem item: the original MediaItem that needs updating.
:return: The original item with more data added to it's properties.
:rtype: MediaItem
"""
Logger.debug('Starting update_video_item for %s (%s)', item.name,
self.channelName)
data = UriHandler.open(item.url,
proxy=self.proxy,
additional_headers=item.HttpHeaders)
json = JsonHelper(data)
video_info = json.get_value("content", "videoInfos")
part = item.create_new_empty_media_part()
if "HLSurlHD" in video_info:
# HLSurlHD=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/
# potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,q50,q60,.mp4.csmil/master.m3u8
for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurlHD"],
self.proxy):
item.complete = True
part.append_media_stream(s, b)
elif "HLSurl" in video_info:
# HLSurl=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/
# potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,.mp4.csmil/master.m3u8
for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurl"],
self.proxy):
item.complete = True
part.append_media_stream(s, b)
if "downloadLink" in video_info:
# downloadLink=http://podcastsource.sf.tv/nps/podcast/10vor10/2015/03/
# 10vor10_20150304_215030_v_podcast_h264_q10.mp4
part.append_media_stream(video_info["downloadLink"], 1000)
return item
def log_on(self, username, password):
""" Peforms the logon of a user.
:param str username: The username
:param str password: The password to use
:returns: a AuthenticationResult with the result of the log on
:rtype: AuthenticationResult
"""
# first we need a random context_id R<10 numbers>
context_id = int(random.random() * 8999999999) + 1000000000
# then we do an initial bootstrap call, which retrieves the `gmid` and `ucid` cookies
url = "https://sso.rtl.nl/accounts.webSdkBootstrap" \
"?apiKey={}" \
"&pageURL=https%3A%2F%2Fwww.rtlxl.nl%2F" \
"&format=json" \
"&callback=gigya.callback" \
"&context=R{}".format(self.api_key, context_id)
init_login = UriHandler.open(url, no_cache=True)
init_data = JsonHelper(init_login)
if init_data.get_value("statusCode") != 200:
Logger.error("Error initiating login")
return AuthenticationResult(None)
# actually do the login request, which requires an async call to retrieve the result
login_url = "https://sso.rtl.nl/accounts.login" \
"?context={0}".format(context_id)
login_data = {
"loginID": username,
"password": password,
# "include": "profile,data",
# "includeUserInfo": "true",
"pageURL": "https://www.rtlxl.nl/profiel",
"format": "json",
# "callback": "gigya.callback",
"context": "R{}".format(context_id),
"targetEnv": "jssdk",
"sessionExpiration": 7776000
}
login_data.update(self.__common_param_dict)
login_response = UriHandler.open(login_url,
data=login_data,
no_cache=True)
# Process the result
authentication_result = self.__extract_session_data(login_response)
authentication_result.existing_login = False
return authentication_result
def extract_json_video(self, data):
""" Performs pre-process actions for data processing.
Accepts an data from the process_folder_list method, BEFORE the items are
processed. Allows setting of parameters (like title etc) for the channel.
Inside this method the <data> could be changed and additional items can
be created.
The return values should always be instantiated in at least ("", []).
:param str data: The retrieve data that was loaded for the current item and URL.
:return: A tuple of the data and a list of MediaItems that were generated.
:rtype: tuple[str|JsonHelper,list[MediaItem]]
"""
Logger.info("Performing Pre-Processing")
items = []
data = Regexer.do_regex(r'window.__DATA__ = ([\w\W]+?});\s*window.__PUSH_STATE__', data)[0]
json_data = JsonHelper(data)
# Get the main content container
main_container = [m for m in json_data.get_value("children") if m["type"] == "MainContainer"]
# Extract seasons
seasons = []
if not self.parentItem.metaData.get("is_season", False):
seasons = [
lst["props"]["items"]
for lst in main_container[0]["children"]
if lst["type"] == "SeasonSelector"
]
if seasons:
seasons = [s for s in seasons[0] if s["url"]]
# Inject them
json_data.json["seasons"] = seasons
# Find the actual
line_lists = [lst for lst in main_container[0]["children"] if lst["type"] == "LineList"]
for line_list in line_lists:
if line_list.get("props", {}).get("type") == "video-guide":
json_data.json = line_list["props"]
# Get the actual full episode list
all_episodes = json_data.get_value("filters", "items", 0, "url")
url_all_episodes = "{}{}".format(self.baseUrl, all_episodes)
data = UriHandler.open(url_all_episodes)
json_data = JsonHelper(data)
# And append seasons again
if seasons:
json_data.json["seasons"] = seasons
return json_data, items
Logger.warning("Cannot extract video items")
return json_data, items
