def update_video_item(self, item): """ Accepts an item. It returns an updated item. Usually retrieves the MediaURL and the Thumb! It should return a completed item. """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) # rtmpt://vrt.flash.streampower.be/een//2011/07/1000_110723_getipt_neefs_wiels_Website_EEN.flv # http://www.een.be/sites/een.be/modules/custom/vrt_video/player/player_4.3.swf # now the mediaurl is derived. First we try WMV data = UriHandler.open(item.url, proxy=self.proxy) part = item.create_new_empty_media_part() if "mediazone.vrt.be" not in item.url: # Extract actual media data video_id = Regexer.do_regex('data-video=[\'"]([^"\']+)[\'"]', data)[0] url = "https://mediazone.vrt.be/api/v1/een/assets/%s" % (video_id, ) data = UriHandler.open(url, proxy=self.proxy) json = JsonHelper(data) urls = json.get_value("targetUrls") for url_info in urls: Logger.trace(url_info) if url_info["type"].lower() != "hls": continue hls_url = url_info["url"] for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy): part.append_media_stream(s, b) item.complete = True return item
def load_channel_data(self, data): """ Adds the channel items to the listing. :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ items = [] now = datetime.datetime.now() try: server_time = UriHandler.open("https://www.svtplay.se/api/server_time", proxy=self.proxy, no_cache=True) server_time_json = JsonHelper(server_time) server_time = server_time_json.get_value("time") except: Logger.error("Error determining server time", exc_info=True) server_time = "%04d-%02d-%02dT%02d:%02d:%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second) data = UriHandler.open( "https://www.svtplay.se/api/channel_page?now=%s" % (server_time, ), proxy=self.proxy) return data, items
def send_files(self, name, file_paths): """ Sends multiple files. :param str|unicode name: Name for the gist/paste. :param list[str|unicode] file_paths: List of file paths. :return: The result of the upload. :rtype: any """ if self.__mode != "gist": raise ValueError("Invalid mode for multiple files") params = { "description": name, "public": False, "files": { # name: { # "content": code # } } } for file_path in file_paths: if not os.path.isfile(file_path): continue code = self.__read_file_bytes(file_path) file_name = os.path.split(file_path) params["files"][file_name[-1]] = {"content": code} headers = { "Content-Type": "application/json" } post_data = JsonHelper.dump(params, pretty_print=False) data = UriHandler.open("https://api.github.com/gists", params=post_data, proxy=self.__proxy, additional_headers=headers) if not data: raise IOError("Error posting Gist to GitHub") json_data = JsonHelper(data) url = json_data.get_value("html_url") if self.__logger: self.__logger.info("Gist: %s", url) # minify with google # POST https://www.googleapis.com/urlshortener/v1/url # Content-Type: application/json shortener = {"longUrl": url} google = "https://www.googleapis.com/urlshortener/v1/url?key=%s" % (self.__apiKey,) google_data = UriHandler.open(google, params=JsonHelper.dump(shortener, False), proxy=self.__proxy, additional_headers={"Content-Type": "application/json"}) google_url = JsonHelper(google_data).get_value("id") if self.__logger: self.__logger.info("Goo.gl: %s", google_url) return google_url
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) if not item.url.endswith(".js"): data = UriHandler.open(item.url, proxy=self.proxy) data_id = Regexer.do_regex(r'data-id="(\d+)"[^>]+data-playout', data) if data_id is None: Logger.warning("Cannot find stream-id for L1 stream.") return item data_url = "https://l1.bbvms.com/p/video/c/{}.json".format(data_id[0]) else: data_url = item.url data = UriHandler.open(data_url, proxy=self.proxy) json = JsonHelper(data, logger=Logger.instance()) Logger.trace(json) base_url = json.get_value("publicationData", "defaultMediaAssetPath") streams = json.get_value("clipData", "assets") item.MediaItemParts = [] part = item.create_new_empty_media_part() for stream in streams: url = stream.get("src", None) if "://" not in url: url = "{}{}".format(base_url, url) bitrate = stream.get("bandwidth", None) if url: part.append_media_stream(url, bitrate) if not item.thumb and json.get_value("thumbnails"): url = json.get_value("thumbnails")[0].get("src", None) if url and "http:/" not in url: url = "%s%s" % (self.baseUrl, url) item.thumb = url item.complete = True return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) meta_data = UriHandler.open(item.url, proxy=self.proxy, referer=self.baseUrl) meta = JsonHelper(meta_data) stream_parts = meta.get_value("feed", "items") for stream_part in stream_parts: stream_url = stream_part["group"]["content"] stream_url = stream_url.replace("{device}", "html5") stream_url = "%s&format=json" % (stream_url, ) stream_data = UriHandler.open(stream_url, proxy=self.proxy) stream = JsonHelper(stream_data) # subUrls = stream.get_value("package", "video", "item", 0, "transcript", 0, "typographic") # NOSONAR part = item.create_new_empty_media_part() # m3u8Url = stream.get_value("package", "video", "item", 0, "rendition", 0, "src") # NOSONAR # for s, b in M3u8.get_streams_from_m3u8(m3u8Url, self.proxy): # item.complete = True # part.append_media_stream(s, b) rtmp_datas = stream.get_value("package", "video", "item", 0, "rendition") for rtmp_data in rtmp_datas: rtmp_url = rtmp_data["src"] rtmp_url = rtmp_url.replace("rtmpe://", "rtmp://") bitrate = rtmp_data["bitrate"] part.append_media_stream(rtmp_url, bitrate) item.complete = True Logger.trace("Media url: %s", item) return item
def authenticate(self, username, password): # Step 1: First initiate an authentication request auth_request = self.__get_authentication_request(username) auth_data = JsonHelper.dump(auth_request) auth_headers = { "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", "Accept-Encoding": "identity", "Content-Type": "application/x-amz-json-1.1" } auth_response = UriHandler.open(self.url, proxy=self.__proxy, params=auth_data, additional_headers=auth_headers) auth_response_json = JsonHelper(auth_response) challenge_parameters = auth_response_json.get_value( "ChallengeParameters") if self.__logger: self.__logger.trace(challenge_parameters) challenge_name = auth_response_json.get_value("ChallengeName") if not challenge_name == "PASSWORD_VERIFIER": if self.__logger: self.__logger.error("Cannot start authentication challenge") return None # Step 2: Respond to the Challenge with a valid ChallengeResponse challenge_request = self.__get_challenge_response_request( challenge_parameters, password) challenge_data = JsonHelper.dump(challenge_request) challenge_headers = { "X-Amz-Target": "AWSCognitoIdentityProviderService.RespondToAuthChallenge", "Content-Type": "application/x-amz-json-1.1" } auth_response = UriHandler.open(self.url, proxy=self.__proxy, params=challenge_data, additional_headers=challenge_headers) auth_response_json = JsonHelper(auth_response) if "message" in auth_response_json.json: self.__logger.error("Error logging in: %s", auth_response_json.get_value("message")) return None, None id_token = auth_response_json.get_value("AuthenticationResult", "IdToken") refresh_token = auth_response_json.get_value("AuthenticationResult", "RefreshToken") return id_token, refresh_token
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) # https://api.viervijfzes.be/content/c58996a6-9e3d-4195-9ecf-9931194c00bf # videoId = item.url.split("/")[-1] # url = "%s/video/v3/embed/%s" % (self.baseUrl, videoId,) url = item.url data = UriHandler.open(url, proxy=self.proxy) return self.__update_video(item, data)
def update_live_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_live_item for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=self.httpHeaders) stream_root = Regexer.do_regex(r'<media href="([^"]+\.isml)', data)[0] Logger.debug("Found Live stream root: %s", stream_root) part = item.create_new_empty_media_part() for s, b in F4m.get_streams_from_f4m(item.url, self.proxy): item.complete = True s = s.replace(".f4m", ".m3u8") part.append_media_stream(s, b) return item
def renew_token(self, refresh_token): """ Sets a new access token on the User using the refresh token. The basic expire time of the refresh token is 30 days: http://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-with-identity-providers.html :param str refresh_token: Token to use for refreshing the authorization token. """ refresh_request = { "AuthParameters": { "REFRESH_TOKEN": refresh_token }, "ClientId": self.client_id, "AuthFlow": "REFRESH_TOKEN" } refresh_headers = { "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", "Content-Type": "application/x-amz-json-1.1" } refresh_request_data = JsonHelper.dump(refresh_request) refresh_response = UriHandler.open(self.url, proxy=self.__proxy, params=refresh_request_data, additional_headers=refresh_headers) refresh_json = JsonHelper(refresh_response) id_token = refresh_json.get_value("AuthenticationResult", "IdToken") return id_token
def update_json_video(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item: %s', item.name) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=self.httpHeaders) json_data = JsonHelper(data) streams = json_data.get_value("formats") if not streams: return item qualities = {"720p": 1600, "480p": 1200, "360p": 500, "other": 0} # , "http-hls": 1500, "3gp-mob01": 300, "flv-web01": 500} part = item.create_new_empty_media_part() urls = [] for stream in streams: url = stream["url"].values()[-1] if url in urls: # duplicate url, ignore continue urls.append(url) # actually process the url if not url.endswith(".m3u8"): part.append_media_stream( url=url, bitrate=qualities.get(stream.get("name", "other"), 0) ) item.complete = True # elif AddonSettings.use_adaptive_stream_add_on(): # content_type, url = UriHandler.header(url, self.proxy) # stream = part.append_media_stream(url, 0) # M3u8.SetInputStreamAddonInput(stream, self.proxy) # item.complete = True else: content_type, url = UriHandler.header(url, self.proxy) for s, b in M3u8.get_streams_from_m3u8(url, self.proxy): item.complete = True part.append_media_stream(s, b) return item
def update_music_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_music_item for %s (%s)', item.name, self.channelName) url, data = item.url.split("?") data = UriHandler.open(url, proxy=self.proxy, params=data, additional_headers=item.HttpHeaders) Logger.trace(data) json_data = JsonHelper(data) url = json_data.get_value("url", fallback=None) if url: item.append_single_stream(url) item.Complete = True return item
def update_video_api_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting UpdateChannelItem for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy) json = JsonHelper(data, logger=Logger.instance()) videos = json.get_value("videoReferences") subtitles = json.get_value("subtitleReferences") Logger.trace(videos) return self.__update_item_from_video_references( item, videos, subtitles)
def add_episode_paging(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ items = [] # we need to create page items. So let's just spoof the paging. Youtube has # a 50 max results per query limit. items_per_page = 50 data = UriHandler.open(self.mainListUri, proxy=self.proxy) xml = xmlhelper.XmlHelper(data) nr_items = xml.get_single_node_content("openSearch:totalResults") for index in range(1, int(nr_items), items_per_page): items.append(self.create_episode_item([index, items_per_page])) # Continue working normal! return data, items
def update_video_item(self, item): data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) media_regex = 'data-media="([^"]+)"' media_info = Regexer.do_regex(media_regex, data)[0] media_info = HtmlEntityHelper.convert_html_entities(media_info) media_info = JsonHelper(media_info) Logger.trace(media_info) # sources part = item.create_new_empty_media_part() # high, web, mobile, url media_sources = media_info.json.get("sources", {}) for quality in media_sources: url = media_sources[quality] if quality == "high": bitrate = 2000 elif quality == "web": bitrate = 800 elif quality == "mobile": bitrate = 400 else: bitrate = 0 part.append_media_stream(url, bitrate) # geoLocRestriction item.isGeoLocked = not media_info.get_value( "geoLocRestriction", fallback="world") == "world" item.complete = True return item
def update_json_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ headers = {} if self.localIP: headers.update(self.localIP) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=headers) video_data = JsonHelper(data) stream_data = video_data.get_value("mediaAssetsOnDemand") if not stream_data: return item use_adaptive = AddonSettings.use_adaptive_stream_add_on() stream_data = stream_data[0] part = item.create_new_empty_media_part() if "hlsUrl" in stream_data: hls_url = stream_data["hlsUrl"] if use_adaptive: stream = part.append_media_stream(hls_url, 0) M3u8.set_input_stream_addon_input(stream, self.proxy, headers=headers) item.complete = True else: for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy, headers=headers): item.complete = True part.append_media_stream(s, b) if "timedTextSubtitlesUrl" in stream_data and stream_data[ "timedTextSubtitlesUrl"]: sub_url = stream_data["timedTextSubtitlesUrl"].replace( ".ttml", ".vtt") sub_url = HtmlEntityHelper.url_decode(sub_url) part.Subtitle = SubtitleHelper.download_subtitle(sub_url, format="webvtt") return item
def update_video_item_json_player(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ data = UriHandler.open(item.url, proxy=self.proxy) streams = Regexer.do_regex(r'label:\s*"([^"]+)",\W*file:\s*"([^"]+)"', data) part = item.create_new_empty_media_part() bitrates = {"720p SD": 1200} for stream in streams: part.append_media_stream(stream[1], bitrates.get(stream[0], 0)) item.complete = True return item
def get_movie_id(self, data): """ Performs pre-process actions for data processing. Accepts an data from the process_folder_list method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). :param str data: The retrieve data that was loaded for the current item and URL. :return: A tuple of the data and a list of MediaItems that were generated. :rtype: tuple[str|JsonHelper,list[MediaItem]] """ Logger.info("Performing Pre-Processing") items = [] movie_id = Regexer.do_regex(r"movietrailers://movie/detail/(\d+)", data)[-1] Logger.debug("Found Movie ID: %s", movie_id) url = "%s/trailers/feeds/data/%s.json" % (self.baseUrl, movie_id) data = UriHandler.open(url, proxy=self.proxy) # set it for logging purposes self.parentItem.url = url Logger.debug("Pre-Processing finished") return data, items
def __send_git_hub_gist(self, name, code): """ Send a file to a Github gist. :param str|unicode name: Name of the logfile paste/gist. :param str code: The content to post. :return: the ID of the gist :rtype: int """ params = { "description": name, "public": False, "files": { name: { "content": code } } } headers = {"Content-Type": "application/json"} post_data = JsonHelper.dump(params, pretty_print=False) data = UriHandler.open("https://api.github.com/gists", params=post_data.encode(), proxy=self.__proxy, additional_headers=headers) if not data: raise IOError("Error posting Gist to GitHub") json_data = JsonHelper(data) url = json_data.get_value("html_url") if self.__logger: self.__logger.info("Gist: %s", url) # minify with google # POST https://www.googleapis.com/urlshortener/v1/url # Content-Type: application/json shortener = {"longUrl": url} google = "https://www.googleapis.com/urlshortener/v1/url?key=%s" % ( self.__apiKey, ) google_data = UriHandler.open( google, params=JsonHelper.dump(shortener, False), proxy=self.__proxy, additional_headers={"Content-Type": "application/json"}) return JsonHelper(google_data).get_value("id")
def __send_paste_bin(self, name, code, expire='1M', paste_format=None, user_key=None): """ Send a file to pastebin.com :param str|unicode name: Name of the logfile paste/gist. :param str code: The content to post. :param str|unicode expire: Expiration time. :param str|unicode paste_format: The format for the file. :param str|unicode user_key: The user API key. :return: The result of the upload. :rtype: any """ if not name: raise ValueError("Name missing") if not code: raise ValueError("No code data specified") params = { 'api_option': 'paste', 'api_paste_private': 1, # 0=public 1=unlisted 2=private 'api_paste_name': name, 'api_paste_expire_date': expire, 'api_dev_key': self.__apiKey, 'api_paste_code': code, } if paste_format: params['api_paste_format'] = paste_format if user_key: params['api_user_key'] = user_key post_params = "" for k in params.keys(): post_params = "{0}&{1}={2}".format( post_params, k, HtmlEntityHelper.url_encode(str(params[k]))) post_params = post_params.lstrip("&") if self.__logger: self.__logger.debug("Posting %d chars to pastebin.com", len(code)) data = UriHandler.open("http://pastebin.com/api/api_post.php", params=post_params, proxy=self.__proxy) if "pastebin.com" not in data: raise IOError(data) if self.__logger: self.__logger.info("PasteBin: %s", data) return data
def update_live_stream_xml(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ data = UriHandler.open(item.url, proxy=self.proxy) xml = parseString(data) stream_xmls = xml.getElementsByTagName("mountpoint") Logger.debug("Found %d streams", len(stream_xmls)) part = item.create_new_empty_media_part() for stream_xml in stream_xmls: server_xml = stream_xml.getElementsByTagName("server")[0] server = server_xml.getElementsByTagName( "ip")[0].firstChild.nodeValue port_node = server_xml.getElementsByTagName("port")[0] port = port_node.firstChild.nodeValue protocol = port_node.attributes["type"].firstChild.nodeValue entry = stream_xml.getElementsByTagName( "mount")[0].firstChild.nodeValue bitrate = int( stream_xml.getElementsByTagName("bitrate") [0].firstChild.nodeValue) transports = stream_xml.getElementsByTagName("transport") for transport in transports: transport_type = transport.firstChild.nodeValue if transport_type == "http": url = "{0}://{1}:{2}/{3}".format(protocol, server, port, entry) elif transport_type == "hls": suffix = transport.attributes[ "mountSuffix"].firstChild.nodeValue url = "{0}://{1}:{2}/{3}{4}".format( protocol, server, port, entry, suffix) else: Logger.debug("Ignoring transport type: %s", transport_type) continue part.append_media_stream(url, bitrate) item.complete = True return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) url = item.url data = UriHandler.open(url, proxy=self.proxy) renditions_url = Regexer.do_regex( r'<media:content[^>]+url=\W([^\'"]+)\W', data)[0] renditions_url = HtmlEntityHelper.strip_amp(renditions_url) rendition_data = UriHandler.open(renditions_url, proxy=self.proxy) video_items = Regexer.do_regex( r'<rendition[^>]+bitrate="(\d+)"[^>]*>\W+<src>([^<]+)<', rendition_data) item.MediaItemParts = [] part = item.create_new_empty_media_part() for video_item in video_items: media_url = self.get_verifiable_video_url(video_item[1].replace( "rtmpe", "rtmp")) part.append_media_stream(media_url, video_item[0]) item.complete = True return item
def update_live_item(self, item): data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) media_regex = 'data-media="([^"]+)"' media_info = Regexer.do_regex(media_regex, data)[0] media_info = HtmlEntityHelper.convert_html_entities(media_info) media_info = JsonHelper(media_info) Logger.trace(media_info) part = item.create_new_empty_media_part() hls_url = media_info.get_value("streamUrl") if hls_url is not None and "m3u8" in hls_url: Logger.debug("Found HLS url for %s: %s", media_info.json["streamName"], hls_url) for s, b in M3u8.get_streams_from_m3u8(hls_url, self.proxy): part.append_media_stream(s, b) item.complete = True else: Logger.debug("No HLS url found for %s. Fetching RTMP Token.", media_info.json["streamName"]) # fetch the token: token_url = "%s/api/media/streaming?streamname=%s" \ % (self.baseUrl, media_info.json["streamName"]) token_data = UriHandler.open(token_url, proxy=self.proxy, additional_headers=item.HttpHeaders, no_cache=True) token_data = JsonHelper(token_data) token = token_data.get_value("token") Logger.debug("Found token '%s' for '%s'", token, media_info.json["streamName"]) rtmp_url = "rtmp://rtmp.rtbf.be/livecast/%s?%s pageUrl=%s tcUrl=rtmp://rtmp.rtbf.be/livecast" \ % (media_info.json["streamName"], token, self.baseUrl) rtmp_url = self.get_verifiable_video_url(rtmp_url) part.append_media_stream(rtmp_url, 0) item.complete = True item.isGeoLocked = not media_info.get_value( "geoLocRestriction", fallback="world") == "world" return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) json = JsonHelper(data) video_info = json.get_value("content", "videoInfos") part = item.create_new_empty_media_part() if "HLSurlHD" in video_info: # HLSurlHD=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/ # potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,q50,q60,.mp4.csmil/master.m3u8 for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurlHD"], self.proxy): item.complete = True part.append_media_stream(s, b) elif "HLSurl" in video_info: # HLSurl=http://srfvodhd-vh.akamaihd.net/i/vod/potzmusig/2015/03/ # potzmusig_20150307_184438_v_webcast_h264_,q10,q20,q30,q40,.mp4.csmil/master.m3u8 for s, b in M3u8.get_streams_from_m3u8(video_info["HLSurl"], self.proxy): item.complete = True part.append_media_stream(s, b) if "downloadLink" in video_info: # downloadLink=http://podcastsource.sf.tv/nps/podcast/10vor10/2015/03/ # 10vor10_20150304_215030_v_podcast_h264_q10.mp4 part.append_media_stream(video_info["downloadLink"], 1000) return item
def __update_m3u8(self, url, part, headers, use_kodi_hls): """ Update a video that has M3u8 streams. :param str url: The URL for the stream. :param MediaItemPart part: The new part that needs updating. :param dict[str,str] headers: The URL headers to use. :param bool use_kodi_hls: Should we use the InputStream Adaptive add-on? """ # first see if there are streams in this file, else check the second location. for s, b in M3u8.get_streams_from_m3u8(url, self.proxy, headers=headers): if use_kodi_hls: strm = part.append_media_stream(url, 0) M3u8.set_input_stream_addon_input(strm, headers=headers) # Only the main M3u8 is needed break else: part.append_media_stream(s, b) if not part.MediaStreams and "manifest.m3u8" in url: Logger.warning( "No streams found in %s, trying alternative with 'master.m3u8'", url) url = url.replace("manifest.m3u8", "master.m3u8") for s, b in M3u8.get_streams_from_m3u8(url, self.proxy, headers=headers): if use_kodi_hls: strm = part.append_media_stream(url, 0) M3u8.set_input_stream_addon_input(strm, headers=headers) # Only the main M3u8 is needed break else: part.append_media_stream(s, b) # check for subs # https://mtgxse01-vh.akamaihd.net/i/201703/13/DCjOLN_1489416462884_427ff3d3_,48,260,460,900,1800,2800,.mp4.csmil/master.m3u8?__b__=300&hdnts=st=1489687185~exp=3637170832~acl=/*~hmac=d0e12e62c219d96798e5b5ef31b11fa848724516b255897efe9808c8a499308b&cc1=name=Svenska%20f%C3%B6r%20h%C3%B6rselskadade~default=no~forced=no~lang=sv~uri=https%3A%2F%2Fsubstitch.play.mtgx.tv%2Fsubtitle%2Fconvert%2Fxml%3Fsource%3Dhttps%3A%2F%2Fcdn-subtitles-mtgx-tv.akamaized.net%2Fpitcher%2F20xxxxxx%2F2039xxxx%2F203969xx%2F20396967%2F20396967-swt.xml%26output%3Dm3u8 # https://cdn-subtitles-mtgx-tv.akamaized.net/pitcher/20xxxxxx/2039xxxx/203969xx/20396967/20396967-swt.xml&output=m3u8 if "uri=" in url and not part.Subtitle: Logger.debug("Extracting subs from M3u8") sub_url = url.rsplit("uri=")[-1] sub_url = HtmlEntityHelper.url_decode(sub_url) sub_data = UriHandler.open(sub_url, proxy=self.proxy) subs = [ line for line in sub_data.split("\n") if line.startswith("http") ] if subs: part.Subtitle = SubtitleHelper.download_subtitle( subs[0], format='webvtt', proxy=self.proxy) return
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ headers = {} if self.localIP: headers.update(self.localIP) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=headers) m3u8_url = Regexer.do_regex('data-file="([^"]+)"', data)[0] part = item.create_new_empty_media_part() if AddonSettings.use_adaptive_stream_add_on(with_encryption=False): stream = part.append_media_stream(m3u8_url, 0) M3u8.set_input_stream_addon_input(stream, proxy=self.proxy, headers=headers) item.complete = True else: for s, b, a in M3u8.get_streams_from_m3u8(m3u8_url, self.proxy, headers=headers, map_audio=True): if a and "-audio" not in s: video_part = s.rsplit("-", 1)[-1] video_part = "-%s" % (video_part, ) s = a.replace(".m3u8", video_part) part.append_media_stream(s, b) item.complete = True return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) xml_data = UriHandler.open(item.url, proxy=self.proxy) # <ref type='adaptive' device='pc' host='http://manifest.us.rtl.nl' href='/rtlxl/network/pc/adaptive/components/videorecorder/27/278629/278630/d009c025-6e8c-3d11-8aba-dc8579373134.ssm/d009c025-6e8c-3d11-8aba-dc8579373134.m3u8' /> m3u8_urls = Regexer.do_regex( "<ref type='adaptive' device='pc' host='([^']+)' href='/([^']+)' />", xml_data) if not m3u8_urls: Logger.warning("No m3u8 data found for: %s", item) return item m3u8_url = "%s/%s" % (m3u8_urls[0][0], m3u8_urls[0][1]) part = item.create_new_empty_media_part() # prevent the "418 I'm a teapot" error part.HttpHeaders[ "user-agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0" # Remove the Range header to make all streams start at the beginning. # Logger.debug("Setting an empty 'Range' http header to force playback at the start of a stream") # part.HttpHeaders["Range"] = '' item.complete = M3u8.update_part_with_m3u8_streams( part, m3u8_url, proxy=self.proxy, headers=part.HttpHeaders, channel=self) return item
def __get_video_streams(self, video_id, part): """ Fetches the video stream for a given videoId @param video_id: (integer) the videoId @param part: (MediaPart) the mediapart to add the streams to @return: (bool) indicating a successfull retrieval """ # hardcoded for now as it does not seem top matter dscgeo = '{"countryCode":"%s","expiry":1446917369986}' % ( self.language.upper(), ) dscgeo = HtmlEntityHelper.url_encode(dscgeo) headers = {"Cookie": "dsc-geo=%s" % (dscgeo, )} # send the data http, nothing, host, other = self.baseUrl.split("/", 3) subdomain, domain = host.split(".", 1) url = "https://secure.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls" \ % (domain, video_id,) data = UriHandler.open(url, proxy=self.proxy, additional_headers=headers, no_cache=True) json = JsonHelper(data) url = json.get_value("hls") if url is None: return False streams_found = False if "?" in url: qs = url.split("?")[-1] else: qs = None for s, b in M3u8.get_streams_from_m3u8(url, self.proxy): # and we need to append the original QueryString if "X-I-FRAME-STREAM" in s: continue streams_found = True if qs is not None: if "?" in s: s = "%s&%s" % (s, qs) else: s = "%s?%s" % (s, qs) part.append_media_stream(s, b) return streams_found
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem NOTE: This is a 100% copy of the chn_vtmbe.Channel.update_html_clip_item """ data = UriHandler.open(item.url) json_data = Regexer.do_regex( r"Drupal\.settings,\s*({[\w\W]+?})\);\s*//-->", data) json_data = JsonHelper(json_data[-1]) video_info = json_data.get_value('medialaan_player', ) video_config = None for key in video_info: Logger.trace("Checking key: %s", key) if "videoConfig" not in video_info[key]: continue video_config = video_info[key]['videoConfig']['video'] break if not video_config: Logger.error("No video info found.") streams = video_config['formats'] for stream in streams: stream_url = stream['url'] if stream['type'] == "mp4": item.append_single_stream(stream_url, 0) item.complete = True return item
def update_video_item(self, item): """ Updates an existing MediaItem with more data. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. :param MediaItem item: the original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug('Starting update_video_item for %s (%s)', item.name, self.channelName) data = UriHandler.open(item.url, proxy=self.proxy, additional_headers=item.HttpHeaders) json = JsonHelper(data) part = item.create_new_empty_media_part() part.Subtitle = NpoStream.get_subtitle(json.get_value("mid"), proxy=self.proxy) for stream in json.get_value("videoStreams"): if not stream["url"].startswith("odi"): part.append_media_stream(stream["url"], stream["bitrate"] / 1000) item.complete = True if item.has_media_item_parts(): return item for s, b in NpoStream.get_streams_from_npo(None, json.get_value("mid"), proxy=self.proxy): item.complete = True part.append_media_stream(s, b) return item
def __update_video_from_mpd(self, item, mpd_info, use_adaptive_with_encryption): """ Updates an existing MediaItem with more data based on an MPD stream. :param dict[str,str] mpd_info: Stream info retrieved from the stream json. :param bool use_adaptive_with_encryption: Do we use the Adaptive InputStream add-on? :param MediaItem item: The original MediaItem that needs updating. :return: The original item with more data added to it's properties. :rtype: MediaItem """ Logger.debug("Updating streams using BrightCove data.") part = item.create_new_empty_media_part() mpd_manifest_url = "https:{0}".format(mpd_info["mediaLocator"]) mpd_data = UriHandler.open(mpd_manifest_url, proxy=self.proxy) subtitles = Regexer.do_regex(r'<BaseURL>([^<]+\.vtt)</BaseURL>', mpd_data) if subtitles: Logger.debug("Found subtitle: %s", subtitles[0]) subtitle = SubtitleHelper.download_subtitle(subtitles[0], proxy=self.proxy, format="webvtt") part.Subtitle = subtitle if use_adaptive_with_encryption: # We can use the adaptive add-on with encryption Logger.info("Using MPD InputStreamAddon") license_url = Regexer.do_regex('licenseUrl="([^"]+)"', mpd_data)[0] token = "Bearer {0}".format(mpd_info["playToken"]) key_headers = {"Authorization": token} license_key = Mpd.get_license_key(license_url, key_headers=key_headers) stream = part.append_media_stream(mpd_manifest_url, 0) Mpd.set_input_stream_addon_input(stream, self.proxy, license_key=license_key) item.complete = True else: XbmcWrapper.show_dialog( LanguageHelper.get_localized_string(LanguageHelper.DrmTitle), LanguageHelper.get_localized_string( LanguageHelper.WidevineLeiaRequired)) return item