def extract_main_list_json(self, data): """ Extracts the main list JSON data from the HTML response. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ data, items = self.add_others(data) start_string = "window.__REDUX_STATE__ = " start_data = data.index(start_string) end_data = data.index("</script><script async=") data = data[start_data + len(start_string):end_data] data = JsonHelper(data) letters = data.get_value("reduxAsyncConnect", "page", "components", 1, "data", "items", 1, "data", "items") for letter_data in letters: letter_data = letter_data["data"] Logger.trace("Processing '%s'", letter_data["title"]) for item in letter_data["items"]: episode = self.create_json_episode_item(item) items.append(episode) return data, items
def update_video_item(self, item): data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) media_regex = 'data-media="([^"]+)"' media_info = Regexer.do_regex(media_regex, data)[0] media_info = HtmlEntityHelper.convert_html_entities(media_info) media_info = JsonHelper(media_info) Logger.trace(media_info) # sources part = item.create_new_empty_media_part() # high, web, mobile, url media_sources = media_info.json.get("sources", {}) for quality in media_sources: url = media_sources[quality] if quality == "high": bitrate = 2000 elif quality == "web": bitrate = 800 elif quality == "mobile": bitrate = 400 else: bitrate = 0 part.append_media_stream(url, bitrate) # geoLocRestriction item.isGeoLocked = not media_info.get_value( "geoLocRestriction", fallback="world") == "world" item.complete = True return item
def __iterate_results(self, url_format, results_per_page=150, max_iterations=10): """ Retrieves the full dataset for a multi-set search action. :param str url_format: The url format with start and count placeholders :param int results_per_page: The maximum results per request :param int max_iterations: The maximum number of iterations :returns A Json response with all results :rtype JsonHelper Url format should be like: https://urplay.se/api/bff/v1/search?product_type=series&rows={}&start={} """ results = None for p in range(0, max_iterations): url = url_format.format(results_per_page, p * results_per_page) data = UriHandler.open(url) json_data = JsonHelper(data) result_items = json_data.get_value("results", fallback=[]) if results is None: results = json_data else: results.json["results"] += result_items if len(result_items) < results_per_page: break return results or ""
def extract_json_episodes(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] data = Regexer.do_regex( r'window.__DATA__ = ([\w\W]+?});\s*window.__PUSH_STATE__', data)[0] json_data = JsonHelper(data) main_container = [ m for m in json_data.get_value("children") if m["type"] == "MainContainer" ] line_list = [ item for item in main_container[0]["children"] if item["type"] == "LineList" ] line_list = line_list[0]["props"] json_data.json = line_list return json_data, items
def __is_already_logged_on(self, username): """ Check if the given user is logged on and sets what packages he/she has. :param str username: :return: Indicator if the user is alreadly logged in :rtype: bool """ me = UriHandler.open("https://disco-api.dplay.se/users/me", proxy=self.proxy, no_cache=True) if UriHandler.instance().status.code >= 300: return False account_data = JsonHelper(me) signed_in_user = account_data.get_value("data", "attributes", "username") if signed_in_user is not None and signed_in_user != username: # Log out UriHandler.open("https://disco-api.dplay.se/logout", data="", proxy=self.proxy, no_cache=True) return False logged_in = not account_data.get_value("data", "attributes", "anonymous") if logged_in: Logger.debug("Already logged in") packages = account_data.get_value("data", "attributes", "packages", fallback=[]) self.__has_premium = "Premium" in packages return True else: return False
def __get_api_url(self, operation, hash_value, variables=None): """ Generates a GraphQL url :param str operation: The operation to use :param str hash_value: The hash of the Query :param dict variables: Any variables to pass :return: A GraphQL string :rtype: str """ extensions = { "persistedQuery": { "version": 1, "sha256Hash": hash_value } } extensions = HtmlEntityHelper.url_encode( JsonHelper.dump(extensions, pretty_print=False)) final_vars = {"order_by": "NAME", "per_page": 1000} if variables: final_vars = variables final_vars = HtmlEntityHelper.url_encode( JsonHelper.dump(final_vars, pretty_print=False)) url = "https://graphql.tv4play.se/graphql?" \ "operationName={}&" \ "variables={}&" \ "extensions={}".format(operation, final_vars, extensions) return url
def __update_video_from_brightcove(self, item, data, use_adaptive_with_encryption): """ Updates an existing MediaItem with more data based on an MPD stream. :param str data: Stream info retrieved from BrightCove. :param bool use_adaptive_with_encryption: Do we use the Adaptive InputStream add-on? :param MediaItem item: The original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ part = item.create_new_empty_media_part() # Then try the new BrightCove JSON bright_cove_regex = '<video[^>]+data-video-id="(?<videoId>[^"]+)[^>]+data-account="(?<videoAccount>[^"]+)' bright_cove_data = Regexer.do_regex( Regexer.from_expresso(bright_cove_regex), data) if not bright_cove_data: Logger.warning("Error updating using BrightCove data: %s", item) return item Logger.info("Found new BrightCove JSON data") bright_cove_url = 'https://edge.api.brightcove.com/playback/v1/accounts/' \ '%(videoAccount)s/videos/%(videoId)s' % bright_cove_data[0] headers = { "Accept": "application/json;pk=BCpkADawqM3ve1c3k3HcmzaxBvD8lXCl89K7XEHiKutxZArg2c5RhwJHJANOwPwS_4o7UsC4RhIzXG8Y69mrwKCPlRkIxNgPQVY9qG78SJ1TJop4JoDDcgdsNrg" } bright_cove_data = UriHandler.open(bright_cove_url, additional_headers=headers) bright_cove_json = JsonHelper(bright_cove_data) streams = [ d for d in bright_cove_json.get_value("sources") if d["container"] == "M2TS" ] # Old filter # streams = filter(lambda d: d["container"] == "M2TS", bright_cove_json.get_value("sources")) if not streams: Logger.warning("Error extracting streams from BrightCove data: %s", item) return item # noinspection PyTypeChecker stream_url = streams[0]["src"] # these streams work better with the the InputStreamAddon because it removes the # "range" http header if use_adaptive_with_encryption: Logger.info("Using InputStreamAddon for playback of HLS stream") strm = part.append_media_stream(stream_url, 0) M3u8.set_input_stream_addon_input(strm) item.complete = True return item for s, b in M3u8.get_streams_from_m3u8(stream_url): item.complete = True part.append_media_stream(s, b) return item
def update_video_item(self, item): """ Accepts an item. It returns an updated item. Usually retrieves the MediaURL and the Thumb! It should return a completed item. """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) # rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv # http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf # now the mediaurl is derived. First we try WMV data = UriHandler.open(item.url) part = item.create_new_empty_media_part() if "mediazone.vrt.be" not in item.url: # Extract actual media data video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0] url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, ) data = UriHandler.open(url) json = JsonHelper(data) urls = json.get_value("targetUrls") for url_info in urls: Logger.trace(url_info) if url_info["type"].lower() != "hls": continue hls_url = url_info["url"] for s, b in M3u8.get_streams_from_m3u8(hls_url): part.append_media_stream(s, b) item.complete = True return item
def __get_api_persisted_url(self, operation, hash_value, variables): # NOSONAR """ Generates a GraphQL url :param str operation: The operation to use :param str hash_value: The hash of the Query :param dict variables: Any variables to pass :return: A GraphQL string :rtype: str """ extensions = { "persistedQuery": { "version": 1, "sha256Hash": hash_value } } extensions = HtmlEntityHelper.url_encode( JsonHelper.dump(extensions, pretty_print=False)) variables = HtmlEntityHelper.url_encode( JsonHelper.dump(variables, pretty_print=False)) url = "https://graph.kijk.nl/graphql?" \ "operationName={}&" \ "variables={}&" \ "extensions={}".format(operation, variables, extensions) return url
def __update_video(self, item, data): if not item.url.startswith("https://api.viervijfzes.be/content/"): regex = 'data-video-*id="([^"]+)' m3u8_url = Regexer.do_regex(regex, data)[-1] # we either have an URL now or an uuid else: m3u8_url = item.url.rsplit("/", 1)[-1] if ".m3u8" not in m3u8_url: Logger.info("Not a direct M3u8 file. Need to log in") url = "https://api.viervijfzes.be/content/%s" % (m3u8_url, ) # We need to log in if not self.loggedOn: self.log_on() # add authorization header authentication_header = { "authorization": self.__idToken, "content-type": "application/json" } data = UriHandler.open(url, additional_headers=authentication_header) json_data = JsonHelper(data) m3u8_url = json_data.get_value("video", "S") # Geo Locked? if "/geo/" in m3u8_url.lower(): # set it for the error statistics item.isGeoLocked = True part = item.create_new_empty_media_part() item.complete = M3u8.update_part_with_m3u8_streams( part, m3u8_url, channel=self, encrypted=False) return item
def extract_day_items(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ items = [] json = JsonHelper(data) page_items = json.get_value('items') for item in page_items: video_item = self.create_json_video_item(item, prepend_serie=True) if video_item: items.append(video_item) return data, items
def pre_process_folder_list(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] if "episode.json" in self.parentItem.url: Logger.debug("Fetching Carousel data") json = JsonHelper(data) data = json.get_value("carousel") Logger.debug("Pre-Processing finished") return data, items
def make_episode_dictionary_array(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] json_data = JsonHelper(data) dict_items = json_data.get_value("items", fallback=[]) for item in dict_items: if item == "banners" or item == "curators": continue items.append(self.create_episode_item(dict_items[item])) Logger.debug("Pre-Processing finished") data = "" return data, items
def extract_page_data(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ items = [] json = JsonHelper(data) data = json.get_value("data") Logger.trace(data) if json.get_value("loadMore", fallback=False): url, page = self.parentItem.url.rsplit("/", 1) url = "{0}/{1}".format(url, int(page) + 1) page_item = MediaItem("{0}".format(int(page) + 2), url) page_item.type = "page" items.append(page_item) return data, items
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) # Get the MZID data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) json_data = Regexer.do_regex(r'<script type="application/ld\+json">(.*?)</script>', data) json_info = JsonHelper(json_data[-1]) video_id = json_info.get_value("video", "@id") publication_id = json_info.get_value("publication", -1, "@id") mzid = "{}${}".format(publication_id, video_id) return self.update_video_for_mzid(item, mzid)
def update_music_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_music_item for %s (%s)', item.name, self.channelName) url, data = item.url.split("?") data = UriHandler.open(url, params=data, additional_headers=item.HttpHeaders) Logger.trace(data) json_data = JsonHelper(data) url = json_data.get_value("url", fallback=None) if url: item.append_single_stream(url) item.Complete = True return item
def renew_token(self, refresh_token): """ Sets a new access token on the User using the refresh token. The basic expire time of the refresh token is 30 days: http://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-with-identity-providers.html :param str refresh_token: Token to use for refreshing the authorization token. """ refresh_request = { "AuthParameters": { "REFRESH_TOKEN": refresh_token }, "ClientId": self.client_id, "AuthFlow": "REFRESH_TOKEN" } refresh_headers = { "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", "Content-Type": "application/x-amz-json-1.1" } refresh_request_data = JsonHelper.dump(refresh_request) refresh_response = UriHandler.open(self.url, proxy=self.__proxy, params=refresh_request_data, additional_headers=refresh_headers) refresh_json = JsonHelper(refresh_response) id_token = refresh_json.get_value("AuthenticationResult", "IdToken") return id_token
def create_single_video_item(self, result_set): """ Creates a MediaItem of type 'video' using the result_set from the regex. This method creates a new MediaItem from the Regular Expression or Json results <result_set>. The method should be implemented by derived classes and are specific to the channel. If the item is completely processed an no further data needs to be fetched the self.complete property should be set to True. If not set to True, the self.update_video_item method is called if the item is focussed or selected for playback. :param str result_set: The result_set of the self.episodeItemRegex :return: A new MediaItem of type 'video' or 'audio' (despite the method's name). :rtype: MediaItem|None """ if self.__hasAlreadyVideoItems: # we already have items, so don't show this one, it will be a duplicate return None result_set = result_set.replace('\\x27', "'") json_data = JsonHelper(result_set) url = self.parentItem.url title = json_data.get_value("name") description = HtmlHelper.to_text(json_data.get_value("description")) item = MediaItem(title, url, type="video") item.description = description return item
def update_live_channel(self, item): """ Updates an existing live stream MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ data = UriHandler.open(item.url, no_cache=True) manifest = JsonHelper(data) if "nonPlayable" in manifest.json and manifest.json["nonPlayable"]: Logger.error("Cannot update Live: %s", item) return item source = manifest.get_value("sourceMedium") if source == "audio": return self.__update_live_audio(item, manifest) else: return self.__update_live_video(item, manifest)
def extract_json(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] json_data = Regexer.do_regex('type="application/json">([^<]+)<', data) if not json_data: Logger.warning("No JSON data found.") return data, items json = JsonHelper(json_data[0]) result = [] for key, value in json.json.items(): result.append(value) value["title"] = key # set new json and return JsonHelper object json.json = result return json, items
def update_json_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ headers = {} if self.localIP: headers.update(self.localIP) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=headers) video_data = JsonHelper(data) stream_data = video_data.get_value("mediaAssetsOnDemand") if not stream_data: return item use_adaptive = AddonSettings.use_adaptive_stream_add_on() stream_data = stream_data[0] part = item.create_new_empty_media_part() if "hlsUrl" in stream_data: hls_url = stream_data["hlsUrl"] if use_adaptive: stream = part.append_media_stream(hls_url, 0) M3u8.set_input_stream_addon_input(stream, self.proxy, headers=headers) item.complete = True else: for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy, headers=headers): item.complete = True part.append_media_stream(s, b) if "timedTextSubtitlesUrl" in stream_data and stream_data[ "timedTextSubtitlesUrl"]: sub_url = stream_data["timedTextSubtitlesUrl"].replace( ".ttml", ".vtt") sub_url = HtmlEntityHelper.url_decode(sub_url) part.Subtitle = SubtitleHelper.download_subtitle(sub_url, format="webvtt") return item
def __get_online_version_from_github(self, include_alpha_beta=False): """ Retrieves the current online version. :param bool include_alpha_beta: should we include alpha/beta releases? :return: Returns the current online version or `None` of no version was found. :rtype: None|Version """ data = self.__uriHandler.open(self.updateUrl, no_cache=True) json_data = JsonHelper(data) version_tag = "tag_name" online_versions = [ r[version_tag].lstrip("v").replace("-", "~") for r in json_data.get_value() if bool(r[version_tag]) and ( not r["prerelease"] or include_alpha_beta) ] if not bool(online_versions): return None max_version = None for online_version_data in online_versions: online_version = Version(online_version_data) if not include_alpha_beta and online_version.buildType is not None: self.__logger.trace("Ignoring %s", online_version) continue self.__logger.trace("Found possible version: %s", online_version) if online_version > max_version: max_version = online_version return max_version
def update_video_api_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting UpdateChannelItem for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy) json = JsonHelper(data, logger=Logger.instance()) videos = json.get_value("videoReferences") subtitles = json.get_value("subtitleReferences") Logger.trace(videos) return self.__update_item_from_video_references( item, videos, subtitles)
def update_live_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ if "icecast" in item.url: item.type = "audio" data = UriHandler.open(item.url) stream_info = JsonHelper(data) url = stream_info.get_value("data") item.url = url return self.update_video_item(item)
def add_categories(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] cat = MediaItem( "\b.: Categorieën :.", "http://m.schooltv.nl/api/v1/categorieen.json?size=100") cat.complete = True cat.dontGroup = True items.append(cat) tips = MediaItem( "\b.: Tips :.", "http://m.schooltv.nl/api/v1/programmas/tips.json?size=100") tips.complete = True tips.dontGroup = True items.append(tips) data = JsonHelper(data) ages = MediaItem("\b.: Leeftijden :.", "") ages.complete = True ages.dontGroup = True for age in ("0-4", "5-6", "7-8", "9-12", "13-15", "16-18"): age_item = MediaItem( "%s Jaar" % (age, ), "http://m.schooltv.nl/api/v1/leeftijdscategorieen/%s/afleveringen.json?" "size=%s&sort=Nieuwste" % (age, self.__PageSize)) age_item.complete = True age_item.dontGroup = True ages.items.append(age_item) # We should list programs instead of videos, so just prefill them here. for program in data.get_value(): if age in program['ageGroups']: age_item.items.append(self.create_episode_item(program)) items.append(ages) Logger.debug("Pre-Processing finished") return data, items
def update_json_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ data = UriHandler.open(item.url) video_data = JsonHelper(data) stream_data = video_data.get_value("playable") if not stream_data: return item part = item.create_new_empty_media_part() for stream_info in stream_data["assets"]: url = stream_info["url"] stream_type = stream_info["format"] if stream_type == "HLS": item.complete = M3u8.update_part_with_m3u8_streams(part, url) else: Logger.warning("Found unknow stream type: %s", stream_type) if "subtitles" not in stream_data or not stream_data["subtitles"]: return item for sub in stream_data["subtitles"]: sub_url = None sub_type = sub["type"] default_sub = sub["defaultOn"] if default_sub: sub_url = sub["webVtt"] sub_type = "webvtt" # set Retrospect type if sub_url: part.Subtitle = SubtitleHelper.download_subtitle( sub_url, format=sub_type) break return item
def update_video_item(self, item): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy) # get the playlist GUID playlist_guids = Regexer.do_regex( "<div[^>]+data-playlist-id='([^']+)'[^>]+></div>", data) if not playlist_guids: # let's try the alternative then (for the new channels) playlist_guids = Regexer.do_regex( 'local_playlist[", -]+([a-f0-9]{20})"', data) playlist_guid = playlist_guids[0] play_list_url = "http://api.mtvnn.com/v2/nl/NL/local_playlists/{}.json?video_format=m3u8".format( playlist_guid) data = UriHandler.open(play_list_url, proxy=self.proxy) from resources.lib.helpers.jsonhelper import JsonHelper from resources.lib.streams.m3u8 import M3u8 json_data = JsonHelper(data) m3u8_url = json_data.get_value("local_playlist_videos", 0, "url") part = item.create_new_empty_media_part() item.complete = M3u8.update_part_with_m3u8_streams(part, m3u8_url, proxy=self.proxy, channel=self, encrypted=True) return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) json = JsonHelper(data) video_info = json.get_value("content", "videoInfos") part = item.create_new_empty_media_part() if "HLSurlHD" in video_info: # HLSurlHD=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/ # potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,q50,q60,.mp4.csmil/master.m3u8 for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurlHD"], self.proxy): item.complete = True part.append_media_stream(s, b) elif "HLSurl" in video_info: # HLSurl=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/ # potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,.mp4.csmil/master.m3u8 for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurl"], self.proxy): item.complete = True part.append_media_stream(s, b) if "downloadLink" in video_info: # downloadLink=http://podcastsource.sf.tv/nps/podcast/10vor10/2015/03/ # 10vor10_20150304_215030_v_podcast_h264_q10.mp4 part.append_media_stream(video_info["downloadLink"], 1000) return item
def log_on(self, username, password): """ Peforms the logon of a user. :param str username: The username :param str password: The password to use :returns: a AuthenticationResult with the result of the log on :rtype: AuthenticationResult """ # first we need a random context_id R<10 numbers> context_id = int(random.random() * 8999999999) + 1000000000 # then we do an initial bootstrap call, which retrieves the `gmid` and `ucid` cookies url = "https://sso.rtl.nl/accounts.webSdkBootstrap" \ "?apiKey={}" \ "&pageURL=https%3A%2F%2Fwww.rtlxl.nl%2F" \ "&format=json" \ "&callback=gigya.callback" \ "&context=R{}".format(self.api_key, context_id) init_login = UriHandler.open(url, no_cache=True) init_data = JsonHelper(init_login) if init_data.get_value("statusCode") != 200: Logger.error("Error initiating login") return AuthenticationResult(None) # actually do the login request, which requires an async call to retrieve the result login_url = "https://sso.rtl.nl/accounts.login" \ "?context={0}".format(context_id) login_data = { "loginID": username, "password": password, # "include": "profile,data", # "includeUserInfo": "true", "pageURL": "https://www.rtlxl.nl/profiel", "format": "json", # "callback": "gigya.callback", "context": "R{}".format(context_id), "targetEnv": "jssdk", "sessionExpiration": 7776000 } login_data.update(self.__common_param_dict) login_response = UriHandler.open(login_url, data=login_data, no_cache=True) # Process the result authentication_result = self.__extract_session_data(login_response) authentication_result.existing_login = False return authentication_result
def extract_json_video(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] data = Regexer.do_regex(r'window.__DATA__ = ([\w\W]+?});\s*window.__PUSH_STATE__', data)[0] json_data = JsonHelper(data) # Get the main content container main_container = [m for m in json_data.get_value("children") if m["type"] == "MainContainer"] # Extract seasons seasons = [] if not self.parentItem.metaData.get("is_season", False): seasons = [ lst["props"]["items"] for lst in main_container[0]["children"] if lst["type"] == "SeasonSelector" ] if seasons: seasons = [s for s in seasons[0] if s["url"]] # Inject them json_data.json["seasons"] = seasons # Find the actual line_lists = [lst for lst in main_container[0]["children"] if lst["type"] == "LineList"] for line_list in line_lists: if line_list.get("props", {}).get("type") == "video-guide": json_data.json = line_list["props"] # Get the actual full episode list all_episodes = json_data.get_value("filters", "items", 0, "url") url_all_episodes = "{}{}".format(self.baseUrl, all_episodes) data = UriHandler.open(url_all_episodes) json_data = JsonHelper(data) # And append seasons again if seasons: json_data.json["seasons"] = seasons return json_data, items Logger.warning("Cannot extract video items") return json_data, items