def CreateVideoItemJson(self, resultSet): """Creates a MediaItem of type 'video' using the resultSet from the regex. Arguments: resultSet : tuple (string) - the resultSet of the self.videoItemRegex Returns: A new MediaItem of type 'video' or 'audio' (despite the method's name) This method creates a new MediaItem from the Regular Expression or Json results <resultSet>. The method should be implemented by derived classes and are specific to the channel. If the item is completely processed an no further data needs to be fetched the self.complete property should be set to True. If not set to True, the self.UpdateVideoItem method is called if the item is focussed or selected for playback. """ Logger.Trace(resultSet) # In some cases the name, posix and description are in the root, in other cases in the # 'episode' node posix = resultSet.get('starts_at', None) image = resultSet.get('image', None) name = resultSet.get('name', None) description = resultSet.get('description', '') # the tips has an extra 'episodes' key if 'episode' in resultSet: Logger.Debug("Found subnode: episodes") # set to episode node data = resultSet['episode'] else: Logger.Warning("No subnode 'episodes' found, trying anyways") data = resultSet # look for better values posix = data.get('broadcasted_at', posix) broadcasted = DateHelper.GetDateFromPosix(posix) description = resultSet.get('description', description) videoId = data.get('whatson_id', None) # try to fetch more name data names = [] name = data.get("name", name) if name: names = [name, ] if "series" in data and "name" in data["series"]: names.insert(0, data["series"]["name"]) # Filter the duplicates title = " - ".join(set(names)) item = mediaitem.MediaItem(title, videoId) item.icon = self.icon item.type = 'video' item.complete = False item.description = description # images = data.get('stills', None) if images: # there were images in the stills item.thumb = images[-1]['url'] elif image: # no stills, or empty, check for image item.thumb = image item.SetDate(broadcasted.year, broadcasted.month, broadcasted.day, broadcasted.hour, broadcasted.minute, broadcasted.second) return item
def __init__(self, channelInfo): """Initialisation of the class. Arguments: channelInfo: ChannelInfo - The channel info object to base this channel on. All class variables should be instantiated here and this method should not be overridden by any derived classes. """ chn_class.Channel.__init__(self, channelInfo) # ============== Actual channel setup STARTS here and should be overwritten from derived classes =============== # The following data was taken from http://playapi.mtgx.tv/v3/channels self.channelId = None self.useOldParsing = False if self.channelCode == "se3": self.mainListUri = "https://www.viafree.se/program/" self.noImage = "tv3seimage.png" self.channelId = ( 1209, # TV4 6000, # MTV 6001, # Comedy Central 7000, # Online Only ??? ) elif self.channelCode == "se6": self.mainListUri = "https://www.viafree.se/program/" self.noImage = "tv6seimage.png" self.channelId = (959, ) elif self.channelCode == "se8": self.mainListUri = "https://www.viafree.se/program/" self.noImage = "tv8seimage.png" self.channelId = (801, ) elif self.channelCode == "se10": self.mainListUri = "https://www.viafree.se/program/" self.noImage = "tv10seimage.png" self.channelId = (5462, ) elif self.channelCode == "sesport": raise NotImplementedError( 'ViaSat sport is not in this channel anymore.') # Danish channels elif self.channelCode == "tv3dk": self.mainListUri = "http://www.viafree.dk/programmer" self.noImage = "tv3noimage.png" # self.channelId = (3687, 6200, 6201) -> show all for now # Norwegian Channels elif self.channelCode == "no3": self.mainListUri = "https://www.viafree.no/programmer" self.noImage = "tv3noimage.png" self.channelId = (1550, 6100, 6101) elif self.channelCode == "no4": self.mainListUri = "https://www.viafree.no/programmer" self.noImage = "viasat4noimage.png" self.channelId = (935, ) elif self.channelCode == "no6": self.mainListUri = "https://www.viafree.no/programmer" self.noImage = "viasat4noimage.png" self.channelId = (1337, ) # These are still using old pages! # EE channels elif self.channelCode == "tv3ee": self.mainListUri = "http://tv3play.tv3.ee/sisu" self.noImage = "tv3noimage.png" self.channelId = (1375, 6301, 6302) self.useOldParsing = True elif self.channelCode == "tv6ee": self.mainListUri = "http://tv3play.tv3.ee/sisu" self.noImage = "tv6seimage.png" self.channelId = (6300, ) self.useOldParsing = True # Lithuanian channels elif self.channelCode == "tv3lt": self.mainListUri = "http://play.tv3.lt/programos" # self.mainListUri = "http://www.tv3play.lt/programos" self.noImage = "tv3ltimage.png" self.channelId = (3000, 6503) self.useOldParsing = True elif self.channelCode == "tv6lt": self.mainListUri = "http://play.tv3.lt/programos" # self.mainListUri = "http://www.tv3play.lt/programos" self.noImage = "tv6ltimage.png" self.channelId = (6501, ) self.useOldParsing = True elif self.channelCode == "tv8lt": self.mainListUri = "http://play.tv3.lt/programos" # self.mainListUri = "http://www.tv3play.lt/programos" self.noImage = "tv8seimage.png" self.channelId = (6502, ) self.useOldParsing = True # Letvian Channel elif self.channelCode == "se3lv": self.mainListUri = "http://tvplay.skaties.lv/parraides" # self.mainListUri = "http://www.tvplay.lv/parraides" self.noImage = "tv3lvimage.png" self.channelId = (1482, 6400, 6401, 6402, 6403, 6404, 6405) self.useOldParsing = True self.baseUrl = self.mainListUri.rsplit("/", 1)[0] self.searchInfo = { "se": ["sok", "Sök"], "ee": ["otsing", "Otsi"], "dk": ["sog", "Søg"], "no": ["sok", "Søk"], "lt": ["paieska", "Paieška"], "lv": ["meklet", "Meklēt"] } # setup the urls self.swfUrl = "http://flvplayer.viastream.viasat.tv/flvplayer/play/swf/MTGXPlayer-1.8.swf" if self.useOldParsing: # the epsiode item regex is based on the channelId's. This is because the website has a filter options and shows # different channels on the same URL. self.episodeItemRegex = 'data-channel-id="(?:%s)"[^>]*>\W+<div class="clip-inner">\W+' \ '<a\W+href="([^"]+)"[^>]*>[\w\W]{0,300}?<img[^>]+data-src="([^"]+)"[^>]*>' \ '[\w\W]{0,200}?<h3[^>]+>([^<]+)' % ('|'.join(map(lambda x: str(x), self.channelId)),) self._AddDataParser(self.mainListUri, matchType=ParserData.MatchExact, preprocessor=self.AddSearch, parser=self.episodeItemRegex, creator=self.CreateEpisodeItem) Logger.Warning("Channel still uses old parsing of episodes.") else: # New JSON page data self._AddDataParser(self.mainListUri, preprocessor=self.ExtractJsonData, matchType=ParserData.MatchExact) self._AddDataParser( self.mainListUri, preprocessor=self.ExtractCategoriesAndAddSearch, json=True, matchType=ParserData.MatchExact, parser=("context", "dispatcher", "stores", "ApplicationStore", "programs"), creator=self.CreateJsonEpisodeItem) # This is the new way, but more complex and some channels have items with missing # category slugs and is not compatible with the old method channels. self.useNewPages = not self.useOldParsing and False if self.useNewPages: self._AddDataParser("*", preprocessor=self.ExtractJsonData) self._AddDataParser( "*", json=True, preprocessor=self.MergeSeasonData, # parser=("context", "dispatcher", "stores", "ContentPageProgramStore", "format", "videos", "0", "program"), # creator=self.CreateJsonVideoItem ) self._AddDataParser("http://playapi.mtgx.tv/", updater=self.UpdateVideoItem) else: self._AddDataParser("*", parser=('_embedded', 'videos'), json=True, preprocessor=self.AddClips, creator=self.CreateVideoItem, updater=self.UpdateVideoItem) self.pageNavigationJson = ("_links", "next") self.pageNavigationJsonIndex = 0 self._AddDataParser("*", json=True, parser=self.pageNavigationJson, creator=self.CreatePageItem) searchRegex = '<a\W+href="[^"]+/(?<url>\d+)"[^>]*>[\w\W]{0,300}?<img[^>]+data-src="' \ '(?<thumburl>[^"]+)"[^>]*>[\w\W]{0,200}?<h3[^>]+>(?<title>[^<]+)' # searchRegex = '<a\W+href="([^"]+)"[^>]*>[\w\W]{0,300}?<img[^>]+data-src="' \ # '([^"]+)"[^>]*>[\w\W]{0,200}?<h3[^>]+>([^<]+)' searchRegex = Regexer.FromExpresso(searchRegex) self._AddDataParser(self.__GetSearchUrl(), parser=searchRegex, creator=self.CreateSearchResult) self._AddDataParser("/api/playClient;isColumn=true;query=", json=True, matchType=ParserData.MatchContains, parser=("data", "formats"), creator=self.CreateJsonEpisodeItem) self._AddDataParser("/api/playClient;isColumn=true;query=", json=True, matchType=ParserData.MatchContains, parser=("data", "clips"), creator=self.CreateJsonVideoItem) self._AddDataParser("/api/playClient;isColumn=true;query=", json=True, matchType=ParserData.MatchContains, parser=("data", "episodes"), creator=self.CreateJsonVideoItem) # =============================================================================================================== # non standard items self.episodeLabel = LanguageHelper.GetLocalizedString( LanguageHelper.EpisodeId) self.seasonLabel = LanguageHelper.GetLocalizedString( LanguageHelper.SeasonId) self.__categories = {} # =============================================================================================================== # Test Cases # No GEO Lock: Extra Extra # GEO Lock: # Multi Bitrate: Glamourama # ====================================== Actual channel setup STOPS here ======================================= return
def GetSingleChannel_old(self, className, channelCode): """Imports a single channel Arguments: className : string - class name of the channel to import. Returns: The channels in the requested class. So that could be more, but they can be distinguished using the channelcode. Returns an empty list if no channels were found. """ if not className: raise ValueError("className should be specified.") Logger.Info("Loading channels for class '%s' and channelCode '%s'", className, channelCode) self.__enabledChannels = [] self.__allChannels = [] self.__validChannels = [] self.__channelVersions = [] channel = None # walk the channel dirs to find the one that has the channel addonPath = self.__GetAddonPath() channelPathStart = "%s.channel" % (Config.addonDir, ) folderToFind = className[4:] # list all add-ons for directory in os.listdir(addonPath): # find the xot ones if channelPathStart in directory and "BUILD" not in directory: channelPath = os.path.join(addonPath, directory) # list the subfolders for the requested folder to find the one we need if folderToFind not in os.listdir(channelPath): continue # we perhaps found it. classPath = os.path.join(channelPath, folderToFind) Logger.Debug("Found possible channel folder in %s", classPath) # check the addon.xml with self.__ParseChannelVersionInfo(path) channelVersion = self.__ParseChannelVersionInfo(channelPath) if channelVersion: self.__channelVersions.append(channelVersion) else: # no info was returned, so we will not include the channel Logger.Warning("Match in %s has incorrect version", classPath) continue # create ChannelInfo objects from the xml file and get the correct ChannelInfo object. It coulde that none # is found and we might need to continue (in case there were duplicate channel names fileName = os.path.join(classPath, "chn_" + folderToFind + ".json") Logger.Debug("Loading info for chn_%s @ %s", folderToFind, fileName) if not os.path.isfile(fileName): Logger.Warning("Could not load %s", fileName) continue cis = ChannelInfo.FromJson(fileName) ci = filter(lambda c: c.moduleName == className and (c.channelCode == channelCode or c.channelCode is channelCode), cis) if not ci or len(ci) > 1: Logger.Warning("Could not load channel with className=%s and channelCode=%s from %s", className, channelCode, fileName) continue ci = ci[0] if self.__IsChannelSetUpdated(ci): # apparently a new channel was found, so we need to do it all Logger.Info("Found a new channel, we need to reload all channels") return self.__ImportChannel(className, channelCode) # What platform are we platform = envcontroller.EnvController.GetPlatform() # check if it is enabled or not if self.__ValidateChannelInfo(ci, platform): return ci.GetChannel() else: continue Logger.Error("No Channel found for class '%s' and channelCode '%s'", className, channelCode) return channel
def DownloadSubtitle(url, fileName="", format='sami', proxy=None): """Downloads a SAMI and stores the SRT in the cache folder Arguments: url : string - URL location of the SAMI file Keyword Arguments: fileName : string - Filename to use to store the subtitle in SRT format. if not specified, an MD5 hash of the URL with .xml extension will be used format : string - defines the source format. Defaults to Sami. Returns: The full patch of the cached SRT file. """ if fileName == "": Logger.Debug( "No filename present, generating filename using MD5 hash of url." ) fileName = "%s.srt" % ( encodinghelper.EncodingHelper.EncodeMD5(url), ) elif not fileName.endswith(".srt"): Logger.Debug("No SRT extension present, appending it.") fileName = "%s.srt" % (fileName, ) srt = "" try: localCompletePath = os.path.join(Config.cacheDir, fileName) # no need to download it again! if os.path.exists(localCompletePath): return localCompletePath Logger.Trace("Opening Subtitle URL") raw = UriHandler.Open(url, proxy=proxy) if raw == "": Logger.Warning( "Empty Subtitle path found. Not setting subtitles.") return "" # try to decode it try: raw = raw.decode() except: Logger.Warning( "Converting input to UTF-8 using 'unicode_escape'") raw = raw.decode('unicode_escape') if format.lower() == 'sami': srt = SubtitleHelper.__ConvertSamiToSrt(raw) elif format.lower() == 'srt': srt = raw elif format.lower() == 'ttml': srt = SubtitleHelper.__ConvertTtmlToSrt(raw) elif format.lower() == 'dcsubtitle': srt = SubtitleHelper.__ConvertDCSubtitleToSrt(raw) elif format.lower() == 'json': srt = SubtitleHelper.__ConvertJsonSubtitleToSrt(raw) else: error = "Uknown subtitle format: %s" % (format, ) raise NotImplementedError(error) f = open(localCompletePath, 'w') f.write(srt) f.close() Logger.Info("Saved SRT as %s", localCompletePath) return localCompletePath except: Logger.Error("Error handling Subtitle file: [%s]", srt, exc_info=True) return ""
def UpdateVideoItem(self, item): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName) data = UriHandler.Open(item.url, proxy=self.proxy, additionalHeaders=item.HttpHeaders) videoId = Regexer.DoRegex('data-video="([^"]+)"', data)[-1] url = "https://mediazone.vrt.be/api/v1/canvas/assets/%s" % (videoId, ) data = UriHandler.Open(url, proxy=self.proxy, additionalHeaders=item.HttpHeaders) json = JsonHelper(data) geoLocked = str(json.GetValue("metaInfo", "allowedRegion").lower()) hideGeoLocked = AddonSettings.HideGeoLockedItemsForLocation(geoLocked) if hideGeoLocked: geoRegion = AddonSettings.HideGeoLockedItemsForLocation( geoLocked, True) Logger.Warning( "Found GEO Locked item for region '%s'. Current region is '%s'", geoLocked, geoRegion) return item part = item.CreateNewEmptyMediaPart() for video in json.GetValue("targetUrls"): videoType = video["type"].lower() url = video["url"] if videoType == "progressive_download": bitrate = 1000 elif videoType == "hls": for s, b in M3u8.GetStreamsFromM3u8(url, self.proxy): # s = self.GetVerifiableVideoUrl(s) part.AppendMediaStream(s, b) continue elif videoType == "rtmp": # url=rtmp://vod.stream.vrt.be/mediazone_canvas/_definst_/mp4:2015/11/mz-ast-79a551d6-2621-4a0f-9af0-a272fb0954db-1/video_1296.mp4 url = url.replace("_definst_/mp4:", "?slist=") bitrate = 1100 else: Logger.Debug("Found unhandled stream type '%s':%s", videoType, url) continue part.AppendMediaStream(url, bitrate) item.complete = True return item
def __init__(self, pluginName, params, handle=0): """Initialises the plugin with given arguments.""" # some constants self.actionDownloadVideo = "downloadVideo".lower() # : Action used to download a video item self.actionFavourites = "favourites".lower() # : Action used to show favorites for a channel self.actionAllFavourites = "allfavourites".lower() # : Action used to show all favorites self.actionRemoveFavourite = "removefromfavourites".lower() # : Action used to remove items from favorites self.actionAddFavourite = "addtofavourites".lower() # : Action used to add items to favorites self.actionPlayVideo = "playvideo".lower() # : Action used to play a video item self.actionUpdateChannels = "updatechannels".lower() # : Action used to update channels self.actionListFolder = "listfolder".lower() # : Action used to list a folder self.actionListCategory = "listcategory" # : Action used to show the channels from a category self.actionConfigureChannel = "configurechannel" # : Action used to configure a channel self.keywordPickle = "pickle".lower() # : Keyword used for the pickle item self.keywordAction = "action".lower() # : Keyword used for the action item self.keywordChannel = "channel".lower() # : Keyword used for the channel self.keywordChannelCode = "channelcode".lower() # : Keyword used for the channelcode self.keywordCategory = "category" # : Keyword used for the category self.keywordRandomLive = "rnd" # : Keyword used for randomizing live items self.pluginName = pluginName self.handle = int(handle) # channel objects self.channelObject = None self.channelFile = "" self.channelCode = None self.contentType = "episodes" self.methodContainer = dict() # : storage for the inspect.getmembers(channel) method. Improves performance # determine the query parameters self.params = self.__GetParameters(params) Logger.Info("*********** Starting %s add-on version %s ***********", Config.appName, Config.version) Logger.Debug("Plugin Params: %s (%s) [handle=%s, name=%s, query=%s]", self.params, len(self.params), self.handle, self.pluginName, params) # are we in session? sessionActive = SessionHelper.IsSessionActive(Logger.Instance()) # fetch some environment settings envCtrl = envcontroller.EnvController(Logger.Instance()) # self.FavouritesEnabled = envCtrl.SQLiteEnabled() self.FavouritesEnabled = not envCtrl.IsPlatform(Environments.Xbox) if not sessionActive: # do add-on start stuff Logger.Info("Add-On start detected. Performing startup actions.") # print the folder structure envCtrl.DirectoryPrinter(Config, AddonSettings) # show notification XbmcWrapper.ShowNotification(None, LanguageHelper.GetLocalizedString(LanguageHelper.StartingAddonId) % ( Config.appName,), fallback=False, logger=Logger) # check for updates if envCtrl.IsPlatform(Environments.Xbox): Updater().AutoUpdate() # check if the repository is available envCtrl.IsInstallMethodValid(Config) # check for cache folder envCtrl.CacheCheck() # do some cache cleanup envCtrl.CacheCleanUp(Config.cacheDir, Config.cacheValidTime) envCtrl.CacheCleanUp(AddonSettings.GetUzgCachePath(), AddonSettings.GetUzgCacheDuration() * 24 * 3600, "xot.*") # create a session SessionHelper.CreateSession(Logger.Instance()) #=============================================================================== # Start the plugin version of progwindow #=============================================================================== if len(self.params) == 0: # Show initial start if not in a session # now show the list if AddonSettings.ShowCategories(): self.ShowCategories() else: self.ShowChannelList() #=============================================================================== # Start the plugin verion of the episode window #=============================================================================== else: try: # Determine what stage we are in. Check that there are more than 2 Parameters if len(self.params) > 1 and self.keywordChannel in self.params: # retrieve channel characteristics self.channelFile = os.path.splitext(self.params[self.keywordChannel])[0] self.channelCode = self.params[self.keywordChannelCode] Logger.Debug("Found Channel data in URL: channel='%s', code='%s'", self.channelFile, self.channelCode) # import the channel channelRegister = ChannelImporter.GetRegister() channel = channelRegister.GetSingleChannel(self.channelFile, self.channelCode) if channel is not None: self.channelObject = channel else: Logger.Critical("None or more than one channels were found, unable to continue.") return # init the channel as plugin self.channelObject.InitChannel() Logger.Info("Loaded: %s", self.channelObject.channelName) elif self.keywordCategory in self.params: # no channel needed. pass elif self.keywordAction in self.params and ( self.params[self.keywordAction] == self.actionAllFavourites or self.params[self.keywordAction] == self.actionRemoveFavourite): # no channel needed for these favourites actions. pass else: Logger.Critical("Error determining Plugin action") return #=============================================================================== # See what needs to be done. #=============================================================================== if self.keywordAction not in self.params: Logger.Critical("Action parameters missing from request. Parameters=%s", self.params) return if self.params[self.keywordAction] == self.actionListCategory: self.ShowChannelList(self.params[self.keywordCategory]) elif self.params[self.keywordAction] == self.actionConfigureChannel: self.__ConfigureChannel(self.channelObject) elif self.params[self.keywordAction] == self.actionFavourites: # we should show the favourites self.ShowFavourites(self.channelObject) elif self.params[self.keywordAction] == self.actionAllFavourites: if self.channelObject is not None: Logger.Warning("We have a self.channelObject with self.actionAllFavourites") self.ShowFavourites(None) elif self.params[self.keywordAction] == self.actionRemoveFavourite: self.RemoveFavourite() elif self.params[self.keywordAction] == self.actionAddFavourite: self.AddFavourite() elif self.params[self.keywordAction] == self.actionListFolder: # channelName and URL is present, Parse the folder self.ProcessFolderList() elif self.params[self.keywordAction] == self.actionPlayVideo: self.PlayVideoItem() elif not self.params[self.keywordAction] == "": self.OnActionFromContextMenu(self.params[self.keywordAction]) else: Logger.Warning("Number of parameters (%s) or parameter (%s) values not implemented", len(self.params), self.params) except: Logger.Critical("Error parsing for add-on", exc_info=True) self.__FetchTextures() return
def CreateVideoItem(self, resultSet): """Creates a MediaItem of type 'video' using the resultSet from the regex. Arguments: resultSet : tuple (string) - the resultSet of the self.videoItemRegex Returns: A new MediaItem of type 'video' or 'audio' (despite the method's name) This method creates a new MediaItem from the Regular Expression results <resultSet>. The method should be implemented by derived classes and are specific to the channel. If the item is completely processed an no further data needs to be fetched the self.complete property should be set to True. If not set to True, the self.UpdateVideoItem method is called if the item is focused or selected for playback. """ Logger.Trace(resultSet) url = "%s%s" % (self.baseUrl, resultSet["url"]) if self.parentItem.url not in url: return None name = resultSet["title"] desc = resultSet.get("description", "") thumb = resultSet["thumburl"] if thumb and not thumb.startswith("http://"): thumb = "%s%s" % (self.baseUrl, thumb) item = mediaitem.MediaItem(name, url) item.thumb = thumb item.description = desc item.icon = self.icon item.type = 'video' item.complete = False try: nameParts = name.rsplit("/", 3) # possibleDateParts = thumb.split("/") if len(nameParts) == 3: Logger.Debug("Found possible date in name: %s", nameParts) year = nameParts[2] if len(year) == 2: year = 2000 + int(year) month = nameParts[1] day = nameParts[0].rsplit(" ", 1)[1] Logger.Trace("%s - %s - %s", year, month, day) item.SetDate(year, month, day) # elif len(possibleDateParts[3]) == 4 and len(possibleDateParts[4]) == 2: # Logger.Debug("Found possible date in name: %s - %s - %s", # possibleDateParts[3], possibleDateParts[4], possibleDateParts[5]) # # year = int(possibleDateParts[3]) # month = int(possibleDateParts[4]) # if len(possibleDateParts[5]) == 2: # day = int(possibleDateParts[5]) # else: # day = 1 # Logger.Trace("%s - %s - %s", year, month, day) # item.SetDate(year, month, day) except: Logger.Warning("Apparently it was not a date :)") return item
def __GetOpener(self, url, proxy=None, userAgent=None, headOnly=False, disableCaching=False, referer=None, additionalHeaders=None, acceptCompression=True): """Get's a urllib2 URL opener with cookie jar Arguments: url : string - The URL to get an opener for Keyword Arguments: proxy : [opt] string - The address and port (proxy.address.ext:port) of a proxy server that should be used. headOnly : [opt] boolean - Indication that only the header is needed. disableCaching : [opt] boolean - Indication to disable the caching. referer : [opt] string - The referer URL additionalHeaders : [opt] dict - A dictionary of additional headers Returns: An urllib2 OpenerDirector object for handling URL requests. """ # create an empty dict, as it cannot be used as a default parameter # http://pythonconquerstheuniverse.wordpress.com/category/python-gotchas/ if not additionalHeaders: additionalHeaders = dict() headHandler = HttpHeadHandler() cacheHandler = None if self.useCaching: if disableCaching: Logger.Info("Disabling caching for this request") else: cacheHandler = cachehttphandler.CacheHttpHandler( self.cacheStore, logger=Logger.Instance()) urlHandlers = [urllib2.HTTPCookieProcessor(self.cookieJar)] if self.ignoreSslErrors: Logger.Warning("Disabling SSL Verification for %s", url) # urlHandlers = [urllib2.HTTPCookieProcessor(self.cookieJar), HTTPSHandlerV3] # noinspection PyProtectedMember,PyTypeChecker urlHandlers.append( urllib2.HTTPSHandler( context=ssl._create_unverified_context())) if proxy is None: pass elif not proxy.UseProxyForUrl(url): Logger.Debug("Not using proxy due to filter mismatch") elif proxy.Scheme == "http": Logger.Debug("Using a http(s) %s", proxy) urlHandlers.append(proxy.GetSmartProxyHandler()) # if there was an http scheme proxy, also add a https one as they will probably work if proxy.Scheme == "http": urlHandlers.append(proxy.GetSmartProxyHandler("https")) elif proxy.Scheme == "dns": Logger.Debug("Using an alternative DNS %s", proxy) # noinspection PyTypeChecker urlHandlers.append(DnsHTTPHandler) # noinspection PyTypeChecker urlHandlers.append(DnsHTTPSHandler) # now we cache the DNS result resolver = DnsQuery(proxy.Proxy) host = resolver.GetHost(url) results = resolver.ResolveAddress(host) Logger.Debug("Resolved DNS %s to %s", host, results) result = resolver.ResolveAddress(host, (1, ))[-1][1] # store it in the cache self.dnsCache[host] = result Logger.Debug("Cached DNS for %s to %s", host, result) # create the opener uriOpener = urllib2.build_opener(*urlHandlers) if headOnly: uriOpener.add_handler(headHandler) if "Content-Type" in additionalHeaders: uriOpener.add_handler( HttpContentTypeFixHandler( additionalHeaders["Content-Type"])) # add the compression handler before the cache in the # chain. That way we store decompressed data and save # cpu time. if acceptCompression and self.useCompression: compressionHandler = HttpCompressionHandler() uriOpener.add_handler(compressionHandler) if cacheHandler: uriOpener.add_handler(cacheHandler) # let's add some headers headers = [] # change the user agent (thanks to VincePirez @ xbmc forums) Logger.Trace(additionalHeaders) if 'User-Agent' in additionalHeaders: Logger.Info("Using UserAgent from AdditionalHeaders: %s", additionalHeaders['User-Agent']) else: if userAgent is None: user_agent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)" else: Logger.Info("Using custom UserAgent for url: %s", userAgent) user_agent = userAgent # user_agent = "XOT/3.0 (compatible; XBMC; U)" # uriOpener.addheaders = [('User-Agent', user_agent)] # headers.append(('User-Agent', user_agent)) additionalHeaders['User-Agent'] = user_agent # add the custom referer if referer is not None: Logger.Info("Adding custom Referer: '%s'", referer) headers.append(('referer', referer)) # if additionalHeaders: -> there always is an user agent for header in additionalHeaders: headers.append((header, additionalHeaders[header])) uriOpener.addheaders = headers return uriOpener
def CreateProgramItem(self, p): """Creates a new MediaItem for a program Arguments: resultSet : list[string] - the resultSet of the self.episodeItemRegex Returns: A new MediaItem of type 'folder' This method creates a new MediaItem from the Regular Expression or Json results <resultSet>. The method should be implemented by derived classes and are specific to the channel. """ # Logger.Trace(p) name = p["title"] # showName = p.get("video_metadata_show", None) videoId = None channelSlug = None homeChannelSlug = None # get some meta data videoInfos = p["taxonomy_items"] for videoInfo in videoInfos: if videoInfo["type"] == "show": videoId = videoInfo["term_id"] elif videoInfo["type"] == "channel": channelSlug = videoInfo["slug"] elif videoInfo["type"] == "home-channel": homeChannelSlug = videoInfo["slug"] if videoId is None: Logger.Warning("Found '%s' without 'term_id'", name) return None # Logger.Trace("Found '%s/%s' with id='%s'", showName or "<noShowName>", name, videoId) if len(self.channelSlugs) > 0 \ and channelSlug not in self.channelSlugs \ and homeChannelSlug not in self.channelSlugs: Logger.Debug("Found show '%s' for channel '%s' needed '%s'", name, channelSlug, self.channelSlugs) return None # now get the items url = "%s/shows/%s/seasons/?show_id=%s&items=%s&sort=episode_number_desc&page=0" \ % (self.baseUrl, videoId, videoId, self.videoPageSize) item = mediaitem.MediaItem(name, url) item.description = p.get("secondary_title") # set the date date = p["modified"] datePart, timePart = date.split(" ") year, month, day = datePart.split("-") # hours, minutes, seconds = timePart.split(":") # item.SetDate(year, month, day, hours, minutes, seconds) item.SetDate(year, month, day) # set the images thumbId = p["image_data"].get("file", None) if thumbId is not None: thumb = "http://a1.res.cloudinary.com/dumrsasw1/image/upload/c_crop,h_901,w_1352,x_72,y_1/c_fill,h_245,w_368/%s" % ( thumbId, ) fanart = "http://a1.res.cloudinary.com/dumrsasw1/image/upload/%s" % ( thumbId, ) item.thumb = thumb item.fanart = fanart item.isPaid = p["content_info"]["package_label"]["value"] != "Free" return item
def UpdateVideoItem(self, item): """ Accepts an item. It returns an updated item. """ Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName) Logger.Trace(item.url) if not item.url.startswith("http://www.bbc.co.uk/mediaselector/"): Logger.Debug("Determining the stream URL") data = UriHandler.Open(item.url, proxy=self.proxy) needle = '"vpid"\W*"([^"]+)"' vid = Regexer.DoRegex(needle, data)[-1] # streamDataUrl = "http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s/" % (vid,) streamDataUrl = "http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s" % (vid,) # streamDataUrl = "http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s" % (vid,) else: streamDataUrl = item.url # this URL is one from the webbrowser but requires a security part. So NOT: # streamDataUrl = "http://open.live.bbc.co.uk/mediaselector/5/select/version # /2.0/mediaset/pc/vpid/%s" % (vid,) # # but: # streamDataUrl = "http://open.live.bbc.co.uk/mediaselector/5/select/version # /2.0/mediaset/pc/vpid/%s/atk/2214e42b5729dcdd012dfb61a3054d39309ccd31/asn/1/ # And I don't know where that one comes from part = item.CreateNewEmptyMediaPart() if True: streamData = UriHandler.Open(streamDataUrl, proxy=self.proxy) else: from debug.router import Router streamData = Router.GetVia("uk", streamDataUrl, self.proxy) connectionDatas = Regexer.DoRegex( '<media bitrate="(\d+)"[^>]+>\W*' '(<connection[^>]+>\W*)' '(<connection[^>]+>\W*)?' '(<connection[^>]+>\W*)?' '(<connection[^>]+>\W*)?</media>', streamData) for connectionData in connectionDatas: # first the bitrate bitrate = connectionData[0] Logger.Trace("Found Media: %s", connectionData) # go through the available connections for connection in connectionData[1:]: if not connection: continue connectionXml = XmlHelper(connection) Logger.Trace("Analyzing Connection: %s", connection) supplier = connectionXml.GetTagAttribute("connection", {"supplier": None}) protocol = connectionXml.GetTagAttribute("connection", {"protocol": None}) transferFormat = connectionXml.GetTagAttribute("connection", {"transferFormat": None}) Logger.Debug("Found connection information:\n" "Protocol: %s\n" "TransferFormat: %s\n" "Supplier: %s\n" "Bitrate: %s", protocol, transferFormat, supplier, bitrate) if protocol.startswith("http"): if transferFormat != "hls": Logger.Debug("Ignoring TransferFormat: %s", transferFormat) continue if "lime" in supplier or "mf_akamai_uk" in supplier: Logger.Debug("Ignoring Supplier: %s", supplier) continue url = connectionXml.GetTagAttribute("connection", {"href": None}) elif protocol.startswith("rtmp"): Logger.Warning("Ignoring RTMP for now") continue else: Logger.Warning("Unknown protocol: %s", protocol) continue # # # port: we take the default one # # determine protocol # protocol = connectionXml.GetTagAttribute("connection", {"protocol": None}) # if protocol == "http": # Logger.Debug("Http stream found, skipping for now.") # continue # # elif protocol == "": # protocol = "rtmp" # Logger.Debug("Found protocol : %s", protocol) # # # now for the non-http version, we need application, authentication, server, file and kind # application = connectionXml.GetTagAttribute("connection", {"application": None}) # if application == "": # application = "ondemand" # Logger.Debug("Found application : %s", application) # # authentication = connectionXml.GetTagAttribute("connection", {"authString": None}) # authentication = htmlentityhelper.HtmlEntityHelper.ConvertHTMLEntities(authentication) # Logger.Debug("Found authentication: %s", authentication) # # server = connectionXml.GetTagAttribute("connection", {"server": None}) # Logger.Debug("Found server : %s", server) # # fileName = connectionXml.GetTagAttribute("connection", {"identifier": None}) # Logger.Debug("Found identifier : %s", fileName) # # kind = connectionXml.GetTagAttribute("connection", {"kind": None}) # Logger.Debug("Found kind : %s", kind) # # Logger.Trace("XML: %s\nProtocol: %s, Server: %s, Application: %s, Authentication: %s, File: %s , Kind: %s", connection, protocol, server, application, authentication, fileName, kind) # if "akamai" in kind: # Logger.Debug("Not including AKAMAI streams") # continue # # url = "%s://%s/%s?%s playpath=%s?%s" % (protocol, server, application, authentication, fileName, authentication) # # Logger.Debug("Creating RTMP for Akamai type\n%s", url) # # elif kind == "limelight": # # for limelight we need to be more specific on what to play # url = "%s://%s/ app=%s?%s tcurl=%s://%s/%s?%s playpath=%s" % ( # protocol, server, application, authentication, protocol, server, application, authentication, # fileName) # Logger.Debug("Creating RTMP for LimeLight type\n%s", url) # else: # # for a none-limelight we just compose a RTMP stream # url = "%s://%s/%s?%s playpath=%s" % (protocol, server, application, authentication, fileName) # Logger.Debug("Creating RTMP for a None-LimeLight type\n%s", url) # url = self.GetVerifiableVideoUrl(url) # if liveStream: # url = "%s live=1" % (url, ) part.AppendMediaStream(url, bitrate) # get the subtitle subtitles = Regexer.DoRegex('<connection href="(http://www.bbc.co.uk/iplayer/subtitles/[^"]+/)([^/]+.xml)"', streamData) if len(subtitles) > 0: subtitle = subtitles[0] subtitleUrl = "%s%s" % (subtitle[0], subtitle[1]) part.Subtitle = subtitlehelper.SubtitleHelper.DownloadSubtitle(subtitleUrl, subtitle[1], "ttml", proxy=self.proxy) item.complete = True Logger.Trace('finishing UpdateVideoItem: %s.', item) return item
def __RetreiveData(self, destHandle, uri, timeOutValue, progressCallback=None, proxy=None, maxBytes=0, params="", referer=None, additionalHeaders=None, noCache=False, blockMultiplier=1): """Open an URL Async using a thread Arguments: uri : string - the URI to download Keyword Arguments: progressCallback : [opt] boolean - should a progress bar be shown proxy : [opt] string - The address and port (proxy.address.ext:port) of a proxy server that should be used. bytes : [opt] integer - the number of bytes to get. params : [opt] string - data to send with the request (open(uri, params)) headers : [opt] dict - a dictionary of additional headers Returns: The data that was retrieved from the URI. """ # init parameters canceled = False timeOut = False error = False srcHandle = None blocksRead = 0 fileSize = 0 charSet = None blockSize = self.blockSize * blockMultiplier try: if uri == "": return error, canceled, "" if uri.startswith("file:"): index = uri.rfind("?") #index = string.rfind(uri, "?") if index > 0: uri = uri[0:index] Logger.Info( "Opening requested uri: %s (callback=%s, timeout=%s)", uri, progressCallback is not None, timeOutValue) self.__DoCallback(progressCallback, 0, blockSize, 0, False) # set the start time in seconds startTime = time.time() # get an opener and handle opener = self.__GetOpener(uri, proxy, disableCaching=noCache, referer=referer, additionalHeaders=additionalHeaders) if params == '': srcHandle = opener.open(uri) else: srcHandle = opener.open(uri, params) # get some metadata Logger.Debug("Determining number of bytes to fetch") data = srcHandle.info() if data.get('Content-length'): fileSize = int(data.get('Content-length')) Logger.Debug('ByteSize is known (fileSize=' + str(fileSize) + ')') else: fileSize = -1 Logger.Debug('ByteSize is unknown') # check for encoding charSet = None try: contentType = data.get('Content-Type') if contentType: Logger.Trace("Found Content-Type header: %s", contentType) charSetNeedle = 'charset=' charSetIndex = contentType.rfind(charSetNeedle) if charSetIndex > 0: charSetEndIndex = contentType.find( ";", charSetIndex) if charSetEndIndex > 0: charSet = contentType[charSetIndex + len(charSetNeedle ):charSetEndIndex] else: charSet = contentType[charSetIndex + len(charSetNeedle):] Logger.Trace("Found Charset HTML Header: %s", charSet) except: charSet = None blocksRead = 0 while True: block = srcHandle.read(blockSize) if block == "": break destHandle.write(block) blocksRead += 1 canceled = self.__DoCallback(progressCallback, blocksRead, blockSize, fileSize, False) if canceled: break if time.time() > startTime + timeOutValue: timeOut = True break if 0 < maxBytes < blocksRead * blockSize: Logger.Info( 'Stopping download because Bytes > maxBytes') break srcHandle.close() except (IncompleteRead, ValueError): # Python 2.6 throws a IncompleteRead on Chuncked data # Python 2.4 throws a ValueError on Chuncked data Logger.Error("IncompleteRead error opening url %s", uri) try: if srcHandle: srcHandle.close() except UnboundLocalError: pass except: Logger.Critical("Error Opening url %s", uri, exc_info=True) error = True try: if srcHandle: srcHandle.close() except UnboundLocalError: pass # we are finished now self.__DoCallback(progressCallback, blocksRead, blockSize, fileSize, True) if timeOut: Logger.Critical( "The URL lookup did not respond within the TimeOut (%s s)", timeOutValue) elif canceled: Logger.Warning("Opening of %s was canceled", uri) elif not error: Logger.Info("Url %s was opened successfully", uri) if self.cookieJarFile: # noinspection PyUnresolvedReferences self.cookieJar.save() return error, canceled, charSet
def GetXBMCPlayList(self, bitrate, updateItemUrls=False, proxy=None): """ Creates a XBMC Playlist containing the MediaItemParts in this MediaItem Keyword Arguments: bitrate : integer - The bitrate of the streams that should be in the playlist. Given in kbps updateItemUrls : [opt] boolean - If specified, the Playlist items will have a path pointing to the actual stream proxy : [opt] ProxyInfo - The proxy to set Returns: a XBMC Playlist for this MediaItem If the Bitrate keyword is omitted the the bitrate is retrieved using the default bitrate settings: """ playList = xbmc.PlayList(xbmc.PLAYLIST_VIDEO) srt = None playListItems = [] if not updateItemUrls: # if we are not using the resolveUrl method, we need to clear the playlist and set the index playList.clear() currentIndex = 0 else: # copy into a list so we can add stuff in between (we can't do that in an # XBMC PlayList) and then create a new playlist item currentIndex = playList.getposition( ) # this is the location at which we are now. if currentIndex < 0: # no items where there, so we can just start at position 0 currentIndex = 0 Logger.Info( "Updating the playlist for item at position %s and trying to preserve other playlist items", currentIndex) for i in range(0, len(playList)): Logger.Trace("Copying playList item %s out of %s", i + 1, len(playList)) playListItems.append((playList[i].getfilename(), playList[i])) startList = reduce( lambda x, y: "%s\n%s" % (x, y[0]), playListItems, "Starting with Playlist Items (%s)" % (len(playListItems), )) Logger.Debug(startList) playList.clear() logText = "Creating playlist for Bitrate: %s kbps\n%s\nSelected Streams:\n" % ( bitrate, self) # for each MediaItemPart get the URL, starting at the current index index = currentIndex for part in self.MediaItemParts: if len(part.MediaStreams) == 0: Logger.Warning("Ignoring empty MediaPart: %s", part) continue # get the playlist item (stream, xbmcItem) = part.GetXBMCPlayListItem( self, bitrate, updateItemUrls=updateItemUrls) logText = "%s\n + %s" % (logText, stream) streamUrl = stream.Url xbmcParams = dict() if proxy: if stream.Downloaded: logText = "%s\n + Not adding proxy as the stream is already downloaded" % ( logText, ) elif proxy.Scheme.startswith( "http") and not stream.Url.startswith("http"): logText = "%s\n + Not adding proxy due to scheme mismatch" % ( logText, ) elif proxy.Scheme == "dns": logText = "%s\n + Not adding DNS proxy for Kodi streams" % ( logText, ) elif not proxy.UseProxyForUrl(streamUrl): logText = "%s\n + Not adding proxy due to filter mismatch" % ( logText, ) else: if AddonSettings.IsMinVersion(17): # See ffmpeg proxy in https://github.com/xbmc/xbmc/commit/60b21973060488febfdc562a415e11cb23eb9764 xbmcItem.setProperty("proxy.host", proxy.Proxy) xbmcItem.setProperty("proxy.port", str(proxy.Port)) xbmcItem.setProperty("proxy.type", proxy.Scheme) if proxy.Username: xbmcItem.setProperty("proxy.user", proxy.Username) if proxy.Password: xbmcItem.setProperty("proxy.password", proxy.Password) logText = "%s\n + Adding (Krypton) %s" % (logText, proxy) else: xbmcParams["HttpProxy"] = proxy.GetProxyAddress() logText = "%s\n + Adding (Pre-Krypton) %s" % ( logText, proxy) # Now add the actual HTTP headers for k in part.HttpHeaders: xbmcParams[k] = HtmlEntityHelper.UrlEncode(part.HttpHeaders[k]) if xbmcParams: xbmcQueryString = reduce( lambda x, y: "%s&%s=%s" % (x, y, xbmcParams[y]), xbmcParams.keys(), "").lstrip("&") Logger.Debug("Adding Kodi Stream parameters: %s\n%s", xbmcParams, xbmcQueryString) streamUrl = "%s|%s" % (stream.Url, xbmcQueryString) if index == currentIndex and index < len(playListItems): # We need to replace the current item. Logger.Trace( "Replacing current Kodi ListItem at Playlist index %s (of %s)", index, len(playListItems)) playListItems[index] = (streamUrl, xbmcItem) else: # We need to add at the current index Logger.Trace("Inserting Kodi ListItem at Playlist index %s", index) playListItems.insert(index, (streamUrl, xbmcItem)) index += 1 # for now we just add the last subtitle, this will not work if each # part has it's own subtitles. srt = part.Subtitle Logger.Info(logText) endList = reduce( lambda x, y: "%s\n%s" % (x, y[0]), playListItems, "Ended with Playlist Items (%s)" % (len(playListItems), )) Logger.Debug(endList) for playListItem in playListItems: playList.add(playListItem[0], playListItem[1]) return playList, srt
def __ImportChannels(self): # , className = None): """Import the available channels This method will: - iterate through the Addons folder and find all the folders name <basename>.channel.<channelname>. - then adds all the subfolders into a list (with paths). - then all paths are added to the system path, so they can be imported. - then read all the chn_<name>.xml metadata files and add the ChannelInfo objects to the self.__channelsToImport - then the channels in the self.__channelsToImport list are instantiated into the self.channels list. """ Logger.Debug("Importing available channels") # import each channelPath. On import, the channelPath will call the RegisterChannel Method try: # clear a possible previous import self.__enabledChannels = [] self.__allChannels = [] self.__validChannels = [] self.__channelVersions = [] # first find all folders with channels that we might need to import channelImport = [] importTimer = StopWatch("ChannelImporter :: importing channels", Logger.Instance()) addonPath = self.__GetAddonPath() channelPathStart = "%s.channel" % (Config.addonDir,) for directory in os.listdir(addonPath): if channelPathStart in directory and "BUILD" not in directory: path = os.path.join(addonPath, directory) channelVersion = self.__ParseChannelVersionInfo(path) if channelVersion: self.__channelVersions.append(channelVersion) else: # no info was returned, so we will not include the channel continue # get all nested channels subDirs = os.listdir(path) channelImport.extend( [os.path.abspath(os.path.join(path, weapon)) for weapon in subDirs]) channelImport.sort() importTimer.Lap("Directories scanned for .channel") # we need to make sure we don't load multiple channel classes and track if we found updates channelsUpdated = False loadedChannels = [] channelsToImport = [] # now import the channels for channelPath in channelImport: if not os.path.isdir(channelPath): continue # determine channelname channelName = os.path.split(channelPath)[-1] if channelName == self.__updateChannelPath: Logger.Trace("Update path found and skipping: %s", channelName) continue # if loadedChannels.count(channelName) > 0: if channelName in loadedChannels: Logger.Warning( "Not loading: chn_%s.xml in %s because there is already a path with " "name '%s' that name loaded", channelName, channelPath, channelName) continue if channelName.startswith("."): continue # now we can continue loadedChannels.append(channelName) fileName = os.path.join(channelPath, "chn_" + channelName + ".json") Logger.Trace("Loading info for chn_%s @ %s", channelName, fileName) if os.path.isfile(fileName): try: ci = ChannelInfo.FromJson(fileName) if len(ci) <= 0: Logger.Warning("No channels found in '%s'", fileName) continue # Add them to the list to import channelsToImport += ci if self.__IsChannelSetUpdated(ci[0]): if not channelsUpdated: # this was the first update found (otherwise channelsUpdated was True) show a message: title = LanguageHelper.GetLocalizedString( LanguageHelper.InitChannelTitle) text = LanguageHelper.GetLocalizedString( LanguageHelper.InitChannelText) XbmcWrapper.ShowNotification(title, text, displayTime=15000) # set the updates found bit channelsUpdated |= True # Initialise the channelset. self.__InitialiseChannelSet(ci[0]) # And perform all first actions for the included channels in the set for channelInfo in ci: self.__FirstTimeChannelActions(channelInfo) except: Logger.Error("Error import chn_%s.json", channelName, exc_info=True) importTimer.Lap() # What platform are we platform = envcontroller.EnvController.GetPlatform() # instantiate the registered channels for channelInfo in channelsToImport: # noinspection PyUnusedLocal isValid = self.__ValidateChannelInfo(channelInfo, platform) # sort the channels self.__enabledChannels.sort() if channelsUpdated: Logger.Info("New or updated channels found. Updating add-on configuration for all " "channels and user agent") AddonSettings.UpdateAddOnSettingsWithChannels(self.__validChannels, Config) AddonSettings.UpdateUserAgent() else: Logger.Debug("No channel changes found. Skipping add-on configuration for channels") # Should we update the channel index? if channelsUpdated or not os.path.isfile(self.__CHANNEL_INDEX): self.__CreateChannelIndex() Logger.Info("Imported %s channels from which %s are enabled", len(self.__allChannels), len(self.__enabledChannels)) importTimer.Stop() except: Logger.Critical("Error loading channel modules", exc_info=True)
def GetSingleChannel(self, className, channelCode): """Imports a single channel Arguments: className : string - class name of the channel to import. Returns: The channels in the requested class. So that could be more, but they can be distinguished using the channelcode. Returns an empty list if no channels were found. """ if not className: raise ValueError("className should be specified.") Logger.Info("Loading channels for class '%s' and channelCode '%s'", className, channelCode) self.__enabledChannels = [] self.__allChannels = [] self.__validChannels = [] self.__channelVersions = [] # noinspection PyUnusedLocal classPath = None channelPath = None classBaseName = className[4:] if os.path.isfile(self.__CHANNEL_INDEX): Logger.Debug("Using ChannelIndex for channel lookup: %s", self.__CHANNEL_INDEX) fd = None try: fd = open(self.__CHANNEL_INDEX) data = fd.read() finally: if fd is not None and not fd.closed: fd.close() channelIndex = JsonHelper(data) classPath = channelIndex.GetValue(className, channelCode or "null") if classPath is not None: if not os.path.isdir(classPath): Logger.Warning("Missing channel class path '%s' found. Rebuilding the ChannelIndex.", classPath) # remove the old one os.remove(self.__CHANNEL_INDEX) # return self.GetSingleChannel(className, channelCode) return self.__ImportChannel(className, channelCode) channelPath = os.path.join(classPath, "..") else: Logger.Warning("Missing ChannelIndex. Rebuilding the ChannelIndex.") return self.__ImportChannel(className, channelCode) # Logger.Warning("Falling back to classic find pattern") # # # walk the channel dirs to find the one that has the channel # addonPath = self.__GetAddonPath() # channelPathStart = "%s.channel" % (Config.addonDir,) # # # list all add-ons # for directory in os.listdir(addonPath): # # find the xot ones # if channelPathStart in directory and "BUILD" not in directory: # channelPath = os.path.join(addonPath, directory) # # # list the subfolders for the requested folder to find the one we need # if classBaseName not in os.listdir(channelPath): # continue # # # we perhaps found it. # classPath = os.path.join(channelPath, classBaseName) if classPath is None: Logger.Error("No Channel found for class '%s' and channelCode '%s'", className, channelCode) return None Logger.Debug("Found possible channel folder in %s", classPath) # check the addon.xml with self.__ParseChannelVersionInfo(path) channelVersion = self.__ParseChannelVersionInfo(channelPath) if channelVersion: self.__channelVersions.append(channelVersion) else: # no info was returned, so we will not include the channel Logger.Error("Match in %s has incorrect version", classPath) return None # create ChannelInfo objects from the xml file and get the correct ChannelInfo object. It coulde that none # is found and we might need to continue (in case there were duplicate channel names fileName = os.path.join(classPath, "chn_" + classBaseName + ".json") Logger.Debug("Loading info for chn_%s @ %s", classBaseName, fileName) if not os.path.isfile(fileName): Logger.Error("Could not load %s", fileName) return None cis = ChannelInfo.FromJson(fileName) ci = filter(lambda c: c.moduleName == className and (c.channelCode == channelCode or c.channelCode is channelCode), cis) if not ci or len(ci) > 1: Logger.Error("Could not load channel with className=%s and channelCode=%s from %s", className, channelCode, fileName) return None ci = ci[0] if self.__IsChannelSetUpdated(ci): # apparently a new channel was found, so we need to do it all Logger.Info("Found a new channel, we need to reload all channels") return self.__ImportChannel(className, channelCode) # What platform are we platform = envcontroller.EnvController.GetPlatform() # check if it is enabled or not if self.__ValidateChannelInfo(ci, platform): return ci.GetChannel() else: Logger.Error("Invalid Channel found for class '%s' and channelCode '%s'", className, channelCode) return None
def ProcessFolderList(self): """Wraps the channel.ProcessFolderList""" Logger.Info("Plugin::ProcessFolderList Doing ProcessFolderList") try: ok = True selectedItem = None if self.keywordPickle in self.params: selectedItem = Pickler.DePickleMediaItem(self.params[self.keywordPickle]) watcher = stopwatch.StopWatch("Plugin ProcessFolderList", Logger.Instance()) episodeItems = self.channelObject.ProcessFolderList(selectedItem) watcher.Lap("Class ProcessFolderList finished") if len(episodeItems) == 0: Logger.Warning("ProcessFolderList returned %s items", len(episodeItems)) ok = self.__ShowEmptyInformation(episodeItems) else: Logger.Debug("ProcessFolderList returned %s items", len(episodeItems)) xbmcItems = [] for episodeItem in episodeItems: if episodeItem.thumb == "": episodeItem.thumb = self.channelObject.noImage if episodeItem.fanart == "": episodeItem.fanart = self.channelObject.fanart if episodeItem.type == 'folder' or episodeItem.type == 'append' or episodeItem.type == "page": action = self.actionListFolder folder = True elif episodeItem.IsPlayable(): action = self.actionPlayVideo folder = False else: Logger.Critical("Plugin::ProcessFolderList: Cannot determine what to add") continue # Get the XBMC item item = episodeItem.GetXBMCItem() # Get the context menu items contextMenuItems = self.__GetContextMenuItems(self.channelObject, item=episodeItem) item.addContextMenuItems(contextMenuItems) # Get the action URL url = self.__CreateActionUrl(self.channelObject, action=action, item=episodeItem) # Add them to the list of XBMC items xbmcItems.append((url, item, folder)) watcher.Lap("Kodi Items generated") # add items but if OK was False, keep it like that ok = ok and xbmcplugin.addDirectoryItems(self.handle, xbmcItems, len(xbmcItems)) watcher.Lap("items send to Kodi") if selectedItem is None: # mainlist item register channel. Statistics.RegisterChannelOpen(self.channelObject, Initializer.StartTime) watcher.Lap("Statistics send") watcher.Stop() self.__AddSortMethodToHandle(self.handle, episodeItems) # set the content xbmcplugin.setContent(handle=self.handle, content=self.contentType) xbmcplugin.endOfDirectory(self.handle, ok) except: Statistics.RegisterError(self.channelObject) XbmcWrapper.ShowNotification(LanguageHelper.GetLocalizedString(LanguageHelper.ErrorId), LanguageHelper.GetLocalizedString(LanguageHelper.ErrorList), XbmcWrapper.Error, 4000) Logger.Error("Plugin::Error Processing FolderList", exc_info=True) xbmcplugin.endOfDirectory(self.handle, False)
def GetChannels(self, includeDisabled=False, **kwargs): # type: (object) -> list """ Retrieves all enabled channels within Retrospect. If updated channels are found, the those channels are indexed and the channel index is rebuild. @type kwargs: here for backward compatibility @return: a list of ChannelInfo objects of enabled channels. """ sw = StopWatch("ChannelIndex.GetChannels Importer", Logger.Instance()) Logger.Info("Fetching all enabled channels.") self.__enabledChannels = [] self.__allChannels = [] self.__validChannels = [] # What platform are we platform = envcontroller.EnvController.GetPlatform() channelsUpdated = False for channelSet in self.__channelIndex[self.__CHANNEL_INDEX_CHANNEL_KEY]: channelSet = self.__channelIndex[self.__CHANNEL_INDEX_CHANNEL_KEY][channelSet] channelSetInfoPath = channelSet[self.__CHANNEL_INDEX_CHANNEL_INFO_KEY] channelSetVersion = channelSet[self.__CHANNEL_INDEX_CHANNEL_VERSION_KEY] # Check if file exists. If not, rebuild index if not os.path.isfile(channelSetInfoPath) and not self.__reindexed: Logger.Warning("Missing channelSet file: %s.", channelSetInfoPath) self.__RebuildIndex() return self.GetChannels() channelInfos = ChannelInfo.FromJson(channelSetInfoPath, channelSetVersion) # Check if the channel was updated if self.__IsChannelSetUpdated(channelInfos[0]): # let's see if the index has already been updated this section, of not, do it and # restart the ChannelRetrieval. if not self.__reindexed: # rebuild and restart Logger.Warning("Re-index channel index due to channelSet update: %s.", channelSetInfoPath) self.__RebuildIndex() return self.GetChannels() else: Logger.Warning("Found updated channelSet: %s.", channelSetInfoPath) if not channelsUpdated: # this was the first update found (otherwise channelsUpdated was True) show a message: title = LanguageHelper.GetLocalizedString(LanguageHelper.InitChannelTitle) text = LanguageHelper.GetLocalizedString(LanguageHelper.InitChannelText) XbmcWrapper.ShowNotification(title, text, displayTime=15000, logger=Logger.Instance()) channelsUpdated |= True # Initialise the channelset. self.__InitialiseChannelSet(channelInfos[0]) # And perform all first actions for the included channels in the set for channelInfo in channelInfos: self.__InitialiseChannel(channelInfo) # Check the channel validity for channelInfo in channelInfos: if not self.__ChannelIsCorrect(channelInfo): continue self.__allChannels.append(channelInfo) # valid channel for this platform ? if not channelInfo.compatiblePlatforms & platform == platform: Logger.Warning("Not loading: %s -> platform '%s' is not compatible.", channelInfo, Environments.Name(platform)) continue self.__validChannels.append(channelInfo) # was the channel disabled? if not (AddonSettings.ShowChannel( channelInfo) and AddonSettings.ShowChannelWithLanguage( channelInfo.language)): Logger.Warning("Not loading: %s -> Channel was disabled from settings.", channelInfo) continue self.__enabledChannels.append(channelInfo) Logger.Debug("Loading: %s", channelInfo) if channelsUpdated: Logger.Info("New or updated channels found. Updating add-on configuration for all channels and user agent.") AddonSettings.UpdateAddOnSettingsWithChannels(self.__validChannels, Config) AddonSettings.UpdateUserAgent() else: Logger.Debug("No channel changes found. Skipping add-on configuration for channels.") # TODO: perhaps we should check that the settings.xml is correct and not broken? self.__enabledChannels.sort() Logger.Info("Fetch a total of %d channels of which %d are enabled.", len(self.__allChannels), len(self.__enabledChannels)) sw.Stop() if includeDisabled: return self.__validChannels return self.__enabledChannels
def PlayVideoItem(self): """Starts the videoitem using a playlist. """ Logger.Debug("Playing videoitem using PlayListMethod") item = None try: item = Pickler.DePickleMediaItem(self.params[self.keywordPickle]) if item.isDrmProtected and AddonSettings.ShowDrmWarning(): Logger.Debug("Showing DRM Warning message") title = LanguageHelper.GetLocalizedString(LanguageHelper.DrmTitle) message = LanguageHelper.GetLocalizedString(LanguageHelper.DrmText) XbmcWrapper.ShowDialog(title, message) elif item.isDrmProtected: Logger.Debug("DRM Warning message disabled by settings") if not item.complete: item = self.channelObject.ProcessVideoItem(item) # validated the updated item if not item.complete or not item.HasMediaItemParts(): Logger.Warning("UpdateVideoItem returned an item that had item.complete = False:\n%s", item) Statistics.RegisterError(self.channelObject, item=item) if not item.HasMediaItemParts(): # the update failed or no items where found. Don't play XbmcWrapper.ShowNotification(LanguageHelper.GetLocalizedString(LanguageHelper.ErrorId), LanguageHelper.GetLocalizedString(LanguageHelper.NoStreamsId), XbmcWrapper.Error) Logger.Warning("Could not start playback due to missing streams. Item:\n%s", item) return playData = self.channelObject.PlayVideoItem(item) Logger.Debug("Continuing playback in plugin.py") if not playData: Logger.Warning("PlayVideoItem did not return valid playdata") return else: playList, srt = playData # Get the Kodi Player instance (let Kodi decide what player, see # http://forum.kodi.tv/showthread.php?tid=173887&pid=1516662#pid1516662) xbmcPlayer = xbmc.Player() # now we force the busy dialog to close, else the video will not play and the # setResolved will not work. xbmc.executebuiltin("Dialog.Close(busydialog)") resolvedUrl = None if item.IsResolvable(): # now set the resolve to the first URL startIndex = playList.getposition() # the current location if startIndex < 0: startIndex = 0 Logger.Info("Playing stream @ playlist index %s using setResolvedUrl method", startIndex) resolvedUrl = playList[startIndex].getfilename() xbmcplugin.setResolvedUrl(self.handle, True, playList[startIndex]) else: # playlist do not use the setResolvedUrl Logger.Info("Playing stream using Playlist method") xbmcPlayer.play(playList) # the set the subtitles showSubs = AddonSettings.UseSubtitle() if srt and (srt != ""): Logger.Info("Adding subtitle: %s and setting showSubtitles to %s", srt, showSubs) XbmcWrapper.WaitForPlayerToStart(xbmcPlayer, logger=Logger.Instance(), url=resolvedUrl) xbmcPlayer.setSubtitles(srt) xbmcPlayer.showSubtitles(showSubs) except: if item: Statistics.RegisterError(self.channelObject, item=item) else: Statistics.RegisterError(self.channelObject) XbmcWrapper.ShowNotification(LanguageHelper.GetLocalizedString(LanguageHelper.ErrorId), LanguageHelper.GetLocalizedString(LanguageHelper.NoPlaybackId), XbmcWrapper.Error) Logger.Critical("Could not playback the url", exc_info=True) return
def UpdateVideoItem(self, item): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName) url = item.url if self.localIP: item.HttpHeaders.update(self.localIP) if ".m3u8" not in item.url: data = UriHandler.Open(url, proxy=self.proxy, additionalHeaders=item.HttpHeaders) json = JsonHelper(data) url = json.GetValue("mediaUrl") if url is None: Logger.Warning("Could not find mediaUrl in %s", item.url) return f4mNeedle = "/manifest.f4m" if f4mNeedle in url: Logger.Info("Found F4m stream. Converting to M3u8.") url = url[:url.index(f4mNeedle)].replace("/z/", "/i/").replace("http:", "https:") url = "%s/master.m3u8" % (url, ) # are there subs? They are added as URL parameter part = item.CreateNewEmptyMediaPart() subMatches = Regexer.DoRegex('https*%3a%2f%2.+master.m3u8', url) if subMatches: subUrl = HtmlEntityHelper.UrlDecode(subMatches[0]) Logger.Info("Item has subtitles: %s", subUrl) subTitle = SubtitleHelper.DownloadSubtitle(subUrl, format="m3u8srt", proxy=self.proxy) if subTitle: part.Subtitle = subTitle for s, b in M3u8.GetStreamsFromM3u8(url, self.proxy, headers=item.HttpHeaders): item.complete = True # s = self.GetVerifiableVideoUrl(s) part.AppendMediaStream(s, b) if self.localIP: part.HttpHeaders.update(self.localIP) return item
def __GetContextMenuItems(self, channel, item=None, favouritesList=False): """Retrieves the context menu items to display Arguments: channel : Channel - The channel from which to get the context menu items. The channel might be None in case of some actions that do not require a channel. Keyword Arguments item : MediaItem - The item to which the context menu belongs. favouritesList : Boolean - Indication that the menu is for the favorites """ contextMenuItems = [] favs = LanguageHelper.GetLocalizedString(LanguageHelper.FavouritesId) allFavs = LanguageHelper.GetLocalizedString(LanguageHelper.AllFavouritesId) # let's put this one on top if item is not None: # add a default enqueue list cmd = "XBMC.Action(Queue)" enqueue = LanguageHelper.GetLocalizedString(LanguageHelper.QueueItemId) contextMenuItems.append(("%s" % (enqueue,), cmd)) cmdUrl = self.__CreateActionUrl(channel, action=self.actionConfigureChannel) cmd = "XBMC.RunPlugin(%s)" % (cmdUrl,) Logger.Trace("Adding command: %s", cmd) title = LanguageHelper.GetLocalizedString(LanguageHelper.ShowChannelSettings) contextMenuItems.append(("Retro: %s" % (title, ), cmd)) if item is None: if self.FavouritesEnabled: # it's just the channel, so only add the favourites cmdUrl = self.__CreateActionUrl(channel, action=self.actionFavourites) cmd = "XBMC.Container.Update(%s)" % (cmdUrl,) # Logger.Trace("Adding command: %s", cmd) channelFavs = LanguageHelper.GetLocalizedString(LanguageHelper.ChannelFavourites) contextMenuItems.append(("Retro: %s" % (channelFavs,), cmd)) cmdUrl = self.__CreateActionUrl(None, action=self.actionAllFavourites) cmd = "XBMC.Container.Update(%s)" % (cmdUrl,) Logger.Trace("Adding command: %s", cmd) contextMenuItems.append(("Retro: %s" % (allFavs, ), cmd)) return contextMenuItems # add a default refresh list cmd = "XBMC.Container.Refresh()" refresh = LanguageHelper.GetLocalizedString(LanguageHelper.RefreshListId) contextMenuItems.append(("Retro: %s" % (refresh,), cmd)) # we have an item if favouritesList: # we have list of favourites cmdUrl = self.__CreateActionUrl(channel, action=self.actionRemoveFavourite, item=item) cmd = "XBMC.Container.Update(%s)" % (cmdUrl,) # Logger.Trace("Adding command: %s", cmd) remove = LanguageHelper.GetLocalizedString(LanguageHelper.RemoveId) fav = LanguageHelper.GetLocalizedString(LanguageHelper.FavouriteId) contextMenuItems.append(("Retro: %s %s" % (remove, fav), cmd)) elif item.type == "folder" and self.FavouritesEnabled: # we need to run RunPlugin here instead of Refresh as we don't want to refresh any lists # the refreshing results in empty lists in XBMC4Xbox. cmdUrl = self.__CreateActionUrl(channel, action=self.actionAddFavourite, item=item) # cmd = "XBMC.RunPlugin(%s)" % (cmdUrl,) cmd = "XBMC.Container.Update(%s)" % (cmdUrl,) # Logger.Trace("Adding command: %s", cmd) addTo = LanguageHelper.GetLocalizedString(LanguageHelper.AddToId) contextMenuItems.append(("Retro: %s %s" % (addTo, favs), cmd)) # if it was a favourites list, don't add the channel methods as they might be from a different channel if channel is None: return contextMenuItems # now we process the other items possibleMethods = self.__GetMembers(channel) # Logger.Debug(possibleMethods) for menuItem in channel.contextMenuItems: # Logger.Debug(menuItem) if menuItem.itemTypes is None or item.type in menuItem.itemTypes: # We don't care for complete here! # if menuItem.completeStatus == None or menuItem.completeStatus == item.complete: # see if the method is available methodAvailable = False for method in possibleMethods: if method == menuItem.functionName: methodAvailable = True # break from the method loop break if not methodAvailable: Logger.Warning("No method for: %s", menuItem) continue cmdUrl = self.__CreateActionUrl(channel, action=menuItem.functionName, item=item) cmd = "XBMC.RunPlugin(%s)" % (cmdUrl,) title = "Retro: %s" % (menuItem.label,) Logger.Trace("Adding command: %s | %s", title, cmd) contextMenuItems.append((title, cmd)) return contextMenuItems
def CreateVideoItem(self, resultSet): """Creates a MediaItem of type 'video' using the resultSet from the regex. Arguments: resultSet : tuple (string) - the resultSet of the self.videoItemRegex Returns: A new MediaItem of type 'video' or 'audio' (despite the method's name) This method creates a new MediaItem from the Regular Expression or Json results <resultSet>. The method should be implemented by derived classes and are specific to the channel. If the item is completely processed an no further data needs to be fetched the self.complete property should be set to True. If not set to True, the self.UpdateVideoItem method is called if the item is focussed or selected for playback. """ Logger.Trace(resultSet) episodeKey = resultSet["episode_key"] if episodeKey: episodeData = self.episodes.get(episodeKey, None) if not episodeData: Logger.Warning("Could not find episodes data for key: %s", episodeKey) return None Logger.Debug("Found Episode Data: %s", episodeData) else: Logger.Debug("No Episode Data Found") episodeData = None title = resultSet["title"] description = None if episodeData: if title: title = "%s - %s" % (episodeData["name"], title) else: title = episodeData["name"] description = episodeData.get("synopsis", None) # tarifs have datetimes # noinspection PyStatementEffect # """ # "ddr_timeframes": [{ # "start": 1382119200, # "stop": 1382378399, # "tariff": 149 # }, # { # "start": 1382378400, # "tariff": 0 # }], # # """ tariffs = resultSet.get("ddr_timeframes") premiumItem = False if tariffs: Logger.Trace(tariffs) for tariff in tariffs: if tariff["tariff"] > 0: start = tariff.get("start", 0) end = tariff.get("stop", 2147483647) start = DateHelper.GetDateFromPosix(start) end = DateHelper.GetDateFromPosix(end) now = datetime.datetime.now() if start < now < end: premiumItem = True Logger.Debug("Found a tariff for this episode: %s - %s: %s", start, end, tariff["tariff"]) break uuid = resultSet["uuid"] url = "http://www.rtl.nl/system/s4m/xldata/ux/%s?context=rtlxl&d=pc&fmt=adaptive&version=3" % (uuid,) # The JSON urls do not yet work # url = "http://www.rtl.nl/system/s4m/vfd/version=1/d=pc/output=json/fun=abstract/uuid=%s/fmt=smooth" % (uuid,) item = mediaitem.MediaItem(title.title(), url) item.type = "video" item.isPaid = premiumItem item.description = description item.thumb = "%s%s" % (self.posterBase, uuid,) station = resultSet.get("station", None) if station: icon = self.largeIconSet.get(station.lower(), None) if icon: Logger.Trace("Setting icon to: %s", icon) item.icon = icon dateTime = resultSet.get("display_date", None) if dateTime: dateTime = DateHelper.GetDateFromPosix(int(dateTime)) item.SetDate(dateTime.year, dateTime.month, dateTime.day, dateTime.hour, dateTime.minute, dateTime.second) return item
def CreateJsonItem(self, resultSet): """Creates a new MediaItem for an episode Arguments: resultSet : list[string] - the resultSet of the self.episodeItemRegex Returns: A new MediaItem of type 'folder' This method creates a new MediaItem from the Regular Expression results <resultSet>. The method should be implemented by derived classes and are specific to the channel. """ Logger.Trace(resultSet) # determine the title programTitle = resultSet.get("programTitle", "") or "" showTitle = resultSet.get("title", "") or "" if showTitle == "" and programTitle != "": title = programTitle elif showTitle != "" and programTitle == "": title = showTitle elif programTitle == "" and showTitle == "": Logger.Warning("Could not find title for item: %s", resultSet) return None elif showTitle != "" and showTitle != programTitle: title = "%s - %s" % (programTitle, showTitle) else: # they are the same title = showTitle if "live" in resultSet and resultSet["live"]: title = "%s (·Live·)" % (title, ) itemType = resultSet.get("contentType") if "contentUrl" in resultSet: url = resultSet["contentUrl"] else: url = resultSet["url"] broadCastDate = resultSet.get("broadcastDate", None) if itemType in ("videoEpisod", "videoKlipp", "singel"): if not url.startswith("/video/") and not url.startswith("/klipp/"): Logger.Warning("Found video item without a /video/ or /klipp/ url.") return None itemType = "video" if "programVersionId" in resultSet: url = "https://www.svt.se/videoplayer-api/video/%s" % (resultSet["programVersionId"],) else: url = "%s%s" % (self.baseUrl, url) else: itemType = "folder" url = "%s%s" % (self.baseUrl, url) item = mediaitem.MediaItem(title, url) item.icon = self.icon item.type = itemType item.isGeoLocked = resultSet.get("onlyAvailableInSweden", False) item.description = resultSet.get("description", "") if "season" in resultSet and "episodeNumber" in resultSet and resultSet["episodeNumber"]: season = int(resultSet["season"]) episode = int(resultSet["episodeNumber"]) if season > 0 and episode > 0: item.name = "s%02de%02d - %s" % (season, episode, item.name) item.SetSeasonInfo(season, episode) # thumb = resultSet.get("imageMedium", self.noImage).replace("/medium/", "/extralarge/") thumb = self.noImage if self.parentItem: thumb = self.parentItem.thumb for imageKey in ("image", "imageMedium", "thumbnailMedium", "thumbnail", "poster"): if imageKey in resultSet and resultSet[imageKey] is not None: thumb = resultSet[imageKey] break item.thumb = self.__GetThumb(thumb or self.noImage) if broadCastDate is not None: if "+" in broadCastDate: broadCastDate = broadCastDate.rsplit("+")[0] timeStamp = DateHelper.GetDateFromString(broadCastDate, "%Y-%m-%dT%H:%M:%S") item.SetDate(*timeStamp[0:6]) return item
def GetLiveStreamsFromNpo(url, cacheDir, proxy=None, headers=None): """ Retrieve NPO Player Live streams from a different number of stream urls. @param url: (String) The url to download @param cacheDir: (String) The cache dir where to find the 'uzg-i.js' file. @param headers: (dict) Possible HTTP Headers @param proxy: (Proxy) The proxy to use for opening Can be used like this: part = item.CreateNewEmptyMediaPart() for s, b in NpoStream.GetStreamsFromNpo(m3u8Url, self.proxy): item.complete = True # s = self.GetVerifiableVideoUrl(s) part.AppendMediaStream(s, b) """ if url.startswith("http://ida.omroep.nl/aapi/"): Logger.Debug( "Already found an IDA data url '%s'. Using it to fetch the streams.", url) # we already have the m3u8 actualStreamData = UriHandler.Open(url, proxy=proxy, additionalHeaders=headers) url = NpoStream.__FetchActualStream(actualStreamData, proxy) elif url.endswith("m3u8"): Logger.Debug( "Found a stream url '%s'. Using a call to IDA to determine the actual streams.", url) hashCode = NpoStream.GetNpoToken(proxy, cacheDir) actualStreamData = UriHandler.Open( "http://ida.omroep.nl/aapi/?stream=%s&token=%s" % (url, hashCode), proxy=proxy, additionalHeaders=headers) url = NpoStream.__FetchActualStream(actualStreamData, proxy) elif url.startswith("http://e.omroep.nl/metadata/"): Logger.Debug( "Found a metadata url '%s'. Determining the actual stream url's", url) jsonData = UriHandler.Open(url, proxy=proxy) json = JsonHelper(jsonData, Logger.Instance()) streams = [] for stream in json.GetValue("streams"): if stream['type'] != "hls": continue url = stream['url'] for k, v in NpoStream.GetLiveStreamsFromNpo(url, cacheDir, proxy=proxy, headers=headers): streams.append((k, v)) return streams else: Logger.Warning("None-stream url found: %s", url) return [] return M3u8.GetStreamsFromM3u8(url, proxy=proxy)
def __RegisterHit(category, action, label, value=None, referer=None, appVersion=None, appId=None): """ Register an event with Google Analytics @param category: String - Name of category to register @param action: String - Name of action to register @param value: String - Value of action to register @param label: String - The label for the event @param value: int - The value for the event (Defaults to None) @param referer: String - The referer (Defaults to None) @param appVersion: String - Version of the channel @param appId: String - ID of the channel See: https://ga-dev-tools.appspot.com/hit-builder/ v=1&t=event&tid=UA-3902785-1&cid=3c8961be-6a53-48f6-bded-d136760ab55f&ec=Test&ea=Test%20Action&el=Test%20%5Blabel)&ev=100 """ try: if not AddonSettings.SendUsageStatistics(): Logger.Debug("Not sending statistics because the configuration does not allow this.") return postData = { "v": 1, "t": "event", "tid": Config.googleAnalyticsId, "cid": AddonSettings.GetClientId(), "ec": HtmlEntityHelper.UrlEncode(category), # "ec": HtmlEntityHelper.UrlEncode("Test"), "ea": HtmlEntityHelper.UrlEncode(HtmlEntityHelper.ConvertHTMLEntities(action)), "el": HtmlEntityHelper.UrlEncode(HtmlEntityHelper.ConvertHTMLEntities(label)), "an": Config.appName } if value is not None: postData["ev"] = value if appVersion is not None and appId is not None: postData["av"] = appVersion postData["aid"] = appId if referer is not None: if "://" not in referer: referer = "http://%s" % (referer,) postData["dr"] = HtmlEntityHelper.UrlEncode(referer) url = "https://www.google-analytics.com/collect" data = "" for k, v in postData.iteritems(): data += "%s=%s&" % (k, v) data = data.rstrip("&") # url = "http://www.rieter.net/net.rieter.xot.usage/%s/%s/?rnd=%s" % (action, value, rnd) Logger.Debug("Sending statistics: %s", data) # now we need something async without caching userAgent = AddonSettings.GetUserAgent() if userAgent: result = UriHandler.Open(url, additionalHeaders={"User-Agent": userAgent}, params=data, noCache=True) else: result = UriHandler.Open(url, params=data, noCache=True) if len(result) > 0: Logger.Debug("Statistics were successfully sent. Content Length: %d", len(result)) else: Logger.Warning("Statistics were not successfully sent") except: # we should never ever fail here Logger.Warning("Cannot send statistics", exc_info=True) return
def __UpdateVideoItem(self, item, episodeId): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated episodeId : String - The episodeId, e.g.: VARA_xxxxxx Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.Trace("Using Generic UpdateVideoItem method") # get the subtitle subTitleUrl = "http://e.omroep.nl/tt888/%s" % (episodeId,) subTitlePath = subtitlehelper.SubtitleHelper.DownloadSubtitle(subTitleUrl, episodeId + ".srt", format='srt', proxy=self.proxy) # we need an hash code hashCode = NpoStream.GetNpoToken(self.proxy, Config.cacheDir) item.MediaItemParts = [] part = item.CreateNewEmptyMediaPart() part.Subtitle = subTitlePath # then we fetch alternative streams locations and start with the non-adapative ones streamsUrls = [] directStreamVideos = AddonSettings.GetUzgCacheDuration() == 0 streamSource = [ "http://ida.omroep.nl/odi/?prid=%s&puboptions=h264_bb,h264_sb,h264_std&adaptive=no&part=1&token=%s" % ( episodeId, hashCode,)] if directStreamVideos: # if we stream, then we first look for adaptive streams Logger.Debug("UZG is configured to streams, so also check for the adaptive streams") streamSource.insert(0, "http://ida.omroep.nl/odi/?prid=%s&puboptions=adaptive&adaptive=yes&part=1&token=%s" % ( episodeId, hashCode,)) else: Logger.Debug("UZG is configured to download. Not going to fetch the adaptive streams") # get the actual stream locations streams: for streamSourceUrl in streamSource: streamUrlData = UriHandler.Open(streamSourceUrl, proxy=self.proxy, noCache=True) streamJson = JsonHelper(streamUrlData, logger=Logger.Instance()) for url in streamJson.GetValue('streams'): Logger.Trace("Going to look for streams in: %s", url) streamsUrls.append(url) # should we cache before playback if not directStreamVideos: part.CanStream = False # now we should now actually go and fetch urls for url in streamsUrls: data = UriHandler.Open(url, proxy=self.proxy) jsonData = JsonHelper(data, logger=Logger.Instance()) # check for errors streamData = jsonData.GetValue() if "errorstring" in streamData: Logger.Warning("Found error response: %s", streamData["errorstring"]) continue # either do m3u8 or hls if "m3u8" in url.lower(): Logger.Trace("Processing M3U8 Json: %s", url) m3u8url = jsonData.GetValue("url") if m3u8url is None: Logger.Warning("Could not find stream in: %s", m3u8url) continue Logger.Trace("Processing M3U8 Streams: %s", m3u8url) for s, b in M3u8.GetStreamsFromM3u8(m3u8url, self.proxy): item.complete = True part.AppendMediaStream(s, b) # if we found an adaptive m3u8, we take that one as it's better Logger.Info("Found M3u8 streams and using those. Stop looking further for non-adaptive ones.") break else: Logger.Trace("Processing HLS: %s", url) if "h264_bb" in url: bitrate = 500 elif "h264_sb" in url: bitrate = 220 elif "h264_std" in url: bitrate = 1000 else: bitrate = None protocol = jsonData.GetValue('protocol') if protocol: url = "%s://%s%s" % (protocol, jsonData.GetValue('server'), jsonData.GetValue('path')) part.AppendMediaStream(url, bitrate=bitrate) else: Logger.Warning("Found UZG Stream without a protocol. Probably a expired page.") if not item.HasMediaItemParts(): Logger.Warning("Apparently no streams were present in the normal places. Trying streams in metadata") # fetch the meta data to get more streams metaUrl = "http://e.omroep.nl/metadata/%s" % (episodeId,) metaData = UriHandler.Open(metaUrl, proxy=self.proxy) metaJson = JsonHelper(metaData, logger=Logger.Instance()) # sometimes there are streams direct in the meta data file directStreams = metaJson.GetValue("streams", fallback=[]) for stream in directStreams: quality = stream.get("kwaliteit", 0) if quality == 1: bitrate = 180 elif quality == 2: bitrate = 1000 elif quality == 3: bitrate = 1500 else: bitrate = 0 if "formaat" in stream and stream["formaat"] == "h264": bitrate += 1 part.AppendMediaStream(stream["url"], bitrate) # now we can get extra info from the data item.description = metaJson.GetValue("info") item.title = metaJson.GetValue('aflevering_titel') station = metaJson.GetValue('streamSense', 'station') if station is None: item.icon = self.icon elif station.startswith('nederland_1'): item.icon = self.GetImageLocation("1large.png") elif station.startswith('nederland_2'): item.icon = self.GetImageLocation("2large.png") elif station.startswith('nederland_3'): item.icon = self.GetImageLocation("3large.png") Logger.Trace("Icon for station %s = %s", station, item.icon) # <image size="380x285" ratio="4:3">http://u.omroep.nl/n/a/2010-12/380x285_boerzoektvrouw_yvon.png</image> thumbUrls = metaJson.GetValue('images') # , {"size": "380x285"}, {"ratio":"4:3"}) Logger.Trace(thumbUrls) if thumbUrls: thumbUrl = thumbUrls[-1]['url'] if "http" not in thumbUrl: thumbUrl = "http://u.omroep.nl/n/a/%s" % (thumbUrl,) else: thumbUrl = self.noImage item.thumb = thumbUrl item.complete = True return item
def ProcessLiveItems(self, data): """ Processes the Live Streams items Arguments: data : string - the retrieve data that was loaded for the current item and URL. Returns: A tuple of the data and a list of MediaItems that were generated. Accepts an data from the ProcessFolderList method, BEFORE the items are processed. Allows setting of parameters (like title etc) for the channel. Inside this method the <data> could be changed and additional items can be created. The return values should always be instantiated in at least ("", []). """ items = [] Logger.Info("Adding Live Streams") # we basically will check for live channels jsonData = JsonHelper(data, logger=Logger.Instance()) liveStreams = jsonData.GetValue() Logger.Trace(liveStreams) if "videos" in liveStreams: Logger.Debug("Multiple streams found") liveStreams = liveStreams["videos"] elif not isinstance(liveStreams, (list, tuple)): Logger.Debug("Single streams found") liveStreams = (liveStreams, ) else: Logger.Debug("List of stream found") liveStreamValue = None for streams in liveStreams: Logger.Debug("Adding live stream") title = streams.get( 'name') or "%s - Live TV" % (self.channelName, ) liveItem = mediaitem.MediaItem(title, self.liveUrl) liveItem.type = 'video' liveItem.complete = True liveItem.icon = self.icon liveItem.thumb = self.noImage liveItem.isLive = True part = liveItem.CreateNewEmptyMediaPart() for stream in streams: Logger.Trace(stream) bitrate = None # if self.liveSelector and stream not in self.liveSelector: # Logger.Warning("Skipping '%s'", stream) # continue # used in Omrop Fryslan if stream == "android": bitrate = 250 url = streams[stream]["videoLink"] elif stream == "iPad": bitrate = 1000 url = streams[stream]["videoLink"] elif stream == "iPhone": bitrate = 250 url = streams[stream]["videoLink"] # used in RTV Utrecht elif stream == "androidLink": bitrate = 250 url = streams[stream] elif stream == "ipadLink": bitrate = 1000 url = streams[stream] elif stream == "iphoneLink": bitrate = 250 url = streams[stream] elif stream == "tabletLink": bitrate = 300 url = streams[stream] # These windows stream won't work # elif stream == "windowsLink": # bitrate = 1200 # url = streams[stream] # elif stream == "wpLink": # bitrate = 1200 # url = streams[stream] elif stream == "name": pass else: Logger.Warning("No url found for type '%s'", stream) if "livestreams.omroep.nl/live/" in url and url.endswith( "m3u8"): Logger.Info("Found NPO Stream, adding ?protection=url") url = "%s?protection=url" % (url, ) if bitrate: part.AppendMediaStream(url, bitrate) if url == liveStreamValue and ".m3u8" in url: # if it was equal to the previous one, assume we have a m3u8. Reset the others. Logger.Info( "Found same M3u8 stream for all streams for this Live channel, using that one: %s", url) liveItem.MediaItemParts = [] liveItem.url = url liveItem.complete = False break elif "playlist.m3u8" in url: # if we have a playlist, use that one. Reset the others. Logger.Info( "Found M3u8 playlist for this Live channel, using that one: %s", url) liveItem.MediaItemParts = [] liveItem.url = url liveItem.complete = False break else: # add it to the possibilities liveStreamValue = url items.append(liveItem) return "", items
def CreateVideoItem(self, resultSet): """ Call base method and then do some more stuff """ item = chn_class.Channel.CreateVideoItem(self, resultSet) # set the POW id item.url = resultSet["url"] item.isPaid = "premium" in resultSet["class"] # TODO: set date try: dateTime = resultSet["date2"].strip().replace(" ", " ").split(" ") # For #933 we check for NOS Journaal if ":" in dateTime[-1] and item.name == "NOS Journaal": item.name = "{0} - {1}".format(item.name, dateTime[-1]) Logger.Trace(dateTime) if dateTime[0].lower() == "gisteren": dateTime = datetime.datetime.now() + datetime.timedelta(days=-1) item.SetDate(dateTime.year, dateTime.month, dateTime.day) elif dateTime[0].lower() == "vandaag": dateTime = datetime.datetime.now() item.SetDate(dateTime.year, dateTime.month, dateTime.day) elif ":" in dateTime[-1]: if dateTime[-2].isalpha(): year = datetime.datetime.now().year dateTime.insert(-1, year) if item.name == "NOS Journaal": item.name = "{0} - {1}".format(item.name, dateTime[-1]) year = int(dateTime[-2]) month = DateHelper.GetMonthFromName(dateTime[-3], language="nl") day = int(dateTime[-4]) stamp = datetime.datetime(year, month, day) if stamp > datetime.datetime.now(): year -= 1 item.SetDate(year, month, day) else: # there is an actual date present if dateTime[0].isalpha(): # first part is ma/di/wo/do/vr/za/zo dateTime.pop(0) # translate the month month = DateHelper.GetMonthFromName(dateTime[1], language="nl") # if the year is missing, let's assume it is this year if ":" in dateTime[2]: dateTime[2] = datetime.datetime.now().year # in the past of future, if future, we need to substract stamp = datetime.datetime(dateTime[2], month, int(dateTime[0])) if stamp > datetime.datetime.now(): dateTime[2] -= 1 item.SetDate(dateTime[2], month, dateTime[0]) except: Logger.Warning("Cannot set date from label: %s", resultSet["date2"], exc_info=True) # 2016-07-05T00:00:00Z dateValue = resultSet.get("date", None) if dateValue: timeStamp = DateHelper.GetDateFromString(dateValue, "%Y-%m-%dT%H:%M:%SZ") item.SetDate(*timeStamp[0:6]) else: Logger.Warning("Cannot set date from 'data-from': %s", resultSet["date"], exc_info=True) return item
def CreateVideoItem(self, resultSet): """Creates a MediaItem of type 'video' using the resultSet from the regex. Arguments: resultSet : tuple (string) - the resultSet of the self.videoItemRegex Returns: A new MediaItem of type 'video' or 'audio' (despite the method's name) This method creates a new MediaItem from the Regular Expression or Json results <resultSet>. The method should be implemented by derived classes and are specific to the channel. If the item is completely processed an no further data needs to be fetched the self.complete property should be set to True. If not set to True, the self.UpdateVideoItem method is called if the item is focussed or selected for playback. """ Logger.Trace(resultSet) drmLocked = False geoBlocked = resultSet["is_geo_blocked"] # hideGeoBloced = AddonSettings().HideGeoLocked() # if geoBlocked and hideGeoBloced: # Logger.Warning("GeoBlocked item") # return None title = resultSet["title"] if ("_links" not in resultSet or "stream" not in resultSet["_links"] or "href" not in resultSet["_links"]["stream"]): Logger.Warning("No streams found for %s", title) return None # the description description = resultSet["description"].strip() # The long version summary = resultSet["summary"].strip() # The short version # Logger.Trace("Comparing:\nDesc: %s\nSumm:%s", description, summary) if description.startswith(summary): pass else: # the descripts starts with the summary. Don't show description = "%s\n\n%s" % (summary, description) videoType = resultSet["type"] if not videoType == "program": title = "%s (%s)" % (title, videoType.title()) elif resultSet["format_position"][ "is_episodic"]: # and resultSet["format_position"]["episode"] != "0": # make sure we show the episodes and seaso # season = int(resultSet["format_position"]["season"]) episode = int(resultSet["format_position"]["episode"]) # name = "s%02de%02d" % (season, episode) webisode = resultSet.get("webisode", False) # if the name had the episode in it, translate it if episode > 0 and not webisode: description = "%s\n\n%s" % (title, description) title = "%s - %s %s %s %s" % ( resultSet["format_title"], self.seasonLabel, resultSet["format_position"]["season"], self.episodeLabel, resultSet["format_position"]["episode"]) else: Logger.Debug( "Found episode number '0' for '%s', using name instead of episode number", title) url = resultSet["_links"]["stream"]["href"] item = mediaitem.MediaItem(title, url) dateInfo = None dateFormat = "%Y-%m-%dT%H:%M:%S" if "broadcasts" in resultSet and len(resultSet["broadcasts"]) > 0: dateInfo = resultSet["broadcasts"][0]["air_at"] Logger.Trace("Date set from 'air_at'") if "playable_from" in resultSet["broadcasts"][0]: startDate = resultSet["broadcasts"][0]["playable_from"] playableFrom = DateHelper.GetDateFromString( startDate[0:-6], dateFormat) playableFrom = datetime.datetime(*playableFrom[0:6]) if playableFrom > datetime.datetime.now(): drmLocked = True elif "publish_at" in resultSet: dateInfo = resultSet["publish_at"] Logger.Trace("Date set from 'publish_at'") if dateInfo is not None: # publish_at=2007-09-02T21:55:00+00:00 info = dateInfo.split("T") dateInfo = info[0] timeInfo = info[1] dateInfo = dateInfo.split("-") timeInfo = timeInfo.split(":") item.SetDate(dateInfo[0], dateInfo[1], dateInfo[2], timeInfo[0], timeInfo[1], 0) item.type = "video" item.complete = False item.icon = self.icon item.isGeoLocked = geoBlocked item.isDrmProtected = drmLocked thumbData = resultSet['_links'].get('image', None) if thumbData is not None: # item.thumbUrl = thumbData['href'].replace("{size}", "thumb") item.thumb = self.__GetThumbImage(thumbData['href']) item.description = description srt = resultSet.get("sami_path") if not srt: srt = resultSet.get("subtitles_webvtt") if srt: Logger.Debug("Storing SRT/WebVTT path: %s", srt) part = item.CreateNewEmptyMediaPart() part.Subtitle = srt return item
def UpdateVideoItem(self, item): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName) # noinspection PyStatementEffect """ <script type="text/javascript">/* <![CDATA[ */ var movieFlashVars = " image=http://assets.ur.se/id/147834/images/1_l.jpg file=/147000-147999/147834-20.mp4 plugins=http://urplay.se/jwplayer/plugins/gapro-1.swf,http://urplay.se/jwplayer/plugins/sharing-2.swf,http://urplay.se/jwplayer/plugins/captions/captions.swf sharing.link=http://urplay.se/147834 gapro.accountid=UA-12814852-8 captions.margin=40 captions.fontsize=11 captions.back=false captions.file=http://undertexter.ur.se/147000-147999/147834-19.tt streamer=rtmp://streaming.ur.se/ondemand autostart=False"; var htmlVideoElementSource = "http://streaming.ur.se/ondemand/mp4:147834-23.mp4/playlist.m3u8?location=SE"; /* //]]> */ </script> """ data = UriHandler.Open(item.url, proxy=self.proxy) # Extract stream JSON data from HTML streams = Regexer.DoRegex(self.mediaUrlRegex, data) jsonData = streams[0] json = JsonHelper(jsonData, logger=Logger.Instance()) Logger.Trace(json.json) item.MediaItemParts = [] part = item.CreateNewEmptyMediaPart() streams = { # No longer used I think "file_flash": 900, "file_mobile": 750, "file_hd": 2000, "file_html5": 850, "file_html5_hd": 2400, 'file_rtmp': 900, 'file_rtmp_hd': 2400, 'file_http_sub': 750, 'file_http': 900, 'file_http_sub_hd': 2400, 'file_http_hd': 2500 } # u'file_rtmp_hd': u'urplay/mp4: 178000-178999/178963-7.mp4', # u'file_rtmp': u'urplay/mp4: 178000-178999/178963-11.mp4', # # u'file_http': u'urplay/_definst_/mp4: 178000-178999/178963-11.mp4/', # u'file_http_sub_hd': u'urplay/_definst_/mp4: 178000-178999/178963-25.mp4/', # u'file_http_sub': u'urplay/_definst_/mp4: 178000-178999/178963-28.mp4/', # u'file_http_hd': u'urplay/_definst_/mp4: 178000-178999/178963-7.mp4/', # generic server information proxy = json.GetValue("streaming_config", "streamer", "redirect") if proxy is None: proxyData = UriHandler.Open( "http://streaming-loadbalancer.ur.se/loadbalancer.json", proxy=self.proxy, noCache=True) proxyJson = JsonHelper(proxyData) proxy = proxyJson.GetValue("redirect") Logger.Trace("Found RTMP Proxy: %s", proxy) rtmpApplication = json.GetValue("streaming_config", "rtmp", "application") Logger.Trace("Found RTMP Application: %s", rtmpApplication) # find all streams for streamType in streams: if streamType not in json.json: Logger.Debug("%s was not found as stream.", streamType) continue bitrate = streams[streamType] streamUrl = json.GetValue(streamType) Logger.Trace(streamUrl) if not streamUrl: Logger.Debug("%s was found but was empty as stream.", streamType) continue #onlySweden = False if streamUrl.startswith( "se/" ) or ":se/" in streamUrl: # or json.GetValue("only_in_sweden"): -> will be in the future onlySweden = True Logger.Warning( "Streams are only available in Sweden: onlySweden=%s", onlySweden) # No need to replace the se/ part. Just log. # streamUrl = streamUrl.replace("se/", "", 1) # although all urls can be handled via RTMP, let's not do that and make the HTTP ones HTTP alwaysRtmp = False if alwaysRtmp or "_rtmp" in streamType: url = "rtmp://%s/%s/?slist=mp4:%s" % (proxy, rtmpApplication, streamUrl) url = self.GetVerifiableVideoUrl(url) elif "_http" in streamType: url = "http://%s/%smaster.m3u8" % (proxy, streamUrl) else: Logger.Warning("Unsupported Stream Type: %s", streamType) continue part.AppendMediaStream(url.strip("/"), bitrate) # get the subtitles captions = json.GetValue("subtitles") subtitle = None for caption in captions: language = caption["label"] default = caption["default"] url = caption["file"] Logger.Debug("Found subtitle language: %s [Default=%s]", language, default) if "Svenska" in language: Logger.Debug("Selected subtitle language: %s", language) fileName = caption["file"] fileName = fileName[fileName.rindex("/") + 1:] + ".srt" subtitle = subtitlehelper.SubtitleHelper.DownloadSubtitle( url, fileName, "ttml", proxy=self.proxy) break part.Subtitle = subtitle item.complete = True return item