def ProcessFolderList(self, item=None): """Process the selected item and get's it's child items using the available dataparsers. Arguments: item : [opt] MediaItem - the selected item Returns: A list of MediaItems that form the childeren of the <item>. Accepts an <item> and returns a list of MediaListems with at least name & url set. The following actions are done: * determining the correct parsers to use * call a pre-processor * parsing the data with the parsers * calling the creators for item creations if the item is NOne, we assume that we are dealing with the first call for this channel and the mainlist uri is used. """ items = [] self.parentItem = item if item is None: Logger.Info("ProcessFolderList :: No item was specified. Assuming it was the main channel list") url = self.mainListUri headers = self.httpHeaders elif len(item.items) > 0: return item.items else: url = item.url headers = item.HttpHeaders if url.startswith("http:") or url.startswith("https:") or url.startswith("file:"): data = UriHandler.Open(url, proxy=self.proxy, additionalHeaders=headers) elif url.startswith("#"): data = "" elif url == "searchSite": Logger.Debug("Starting to search") return self.SearchSite() else: Logger.Debug("Unknown URL format. Setting data to ''") data = "" # Determine the handlers and process dataParsers = self.__GetDataParsers(url) # first check if there is a generic pre-processor preProcs = filter(lambda p: p.IsGenericPreProcessor(), dataParsers) numPreProcs = len(preProcs) Logger.Trace("Processing %s Generic Pre-Processors DataParsers", numPreProcs) if numPreProcs > 1: # warn for strange results if more than 1 generic pre-processor is present. Logger.Warning("More than one Generic Pre-Processor is found (%s). They are being processed in the " "order that Python likes which might result in unexpected result.", numPreProcs) for dataParser in preProcs: # remove it from the list dataParsers.remove(dataParser) # and process it Logger.Debug("Processing %s", dataParser) (data, preItems) = dataParser.PreProcessor(data) items += preItems # The the other handlers Logger.Trace("Processing %s Normal DataParsers", len(dataParsers)) handlerJson = None for dataParser in dataParsers: Logger.Debug("Processing %s", dataParser) # Check for preprocessors if dataParser.PreProcessor: Logger.Debug("Processing DataParser.PreProcessor") (handlerData, preItems) = dataParser.PreProcessor(data) items += preItems else: handlerData = data Logger.Debug("Processing DataParser.Parser") if dataParser.Parser is None or (dataParser.Parser == "" and not dataParser.IsJson): if dataParser.Creator: Logger.Warning("No <parser> found for %s. Skipping.", dataParser.Creator) continue if dataParser.IsJson: if handlerJson is None: # Cache the json requests to improve performance Logger.Trace("Caching JSON results for Dataparsing") handlerJson = JsonHelper(handlerData, Logger.Instance()) Logger.Trace(dataParser.Parser) parserResults = handlerJson.GetValue(fallback=[], *dataParser.Parser) if not isinstance(parserResults, (tuple, list)): # if there is just one match, return that as a list parserResults = [parserResults] else: parserResults = Regexer.DoRegex(dataParser.Parser, handlerData) Logger.Debug("Processing DataParser.Creator for %s items", len(parserResults)) for parserResult in parserResults: handlerItem = dataParser.Creator(parserResult) if handlerItem is not None: items.append(handlerItem) # should we exclude DRM/GEO? hideGeoLocked = AddonSettings.HideGeoLockedItemsForLocation(self.language) hideDrmProtected = AddonSettings.HideDrmItems() hidePremium = AddonSettings.HidePremiumItems() hideFolders = AddonSettings.HideRestrictedFolders() typeToExclude = None if not hideFolders: typeToExclude = "folder" oldCount = len(items) if hideDrmProtected: Logger.Debug("Hiding DRM items") items = filter(lambda i: not i.isDrmProtected or i.type == typeToExclude, items) if hideGeoLocked: Logger.Debug("Hiding GEO Locked items due to GEO region: %s", self.language) items = filter(lambda i: not i.isGeoLocked or i.type == typeToExclude, items) if hidePremium: Logger.Debug("Hiding Premium items") items = filter(lambda i: not i.isPaid or i.type == typeToExclude, items) # items = filter(lambda i: not i.isPaid or i.type == "folder", items) if len(items) != oldCount: Logger.Info("Hidden %s items due to DRM/GEO/Premium filter (Hide Folders=%s)", oldCount - len(items), hideFolders) # Check for grouping or not limit = AddonSettings.GetListLimit() folderItems = filter(lambda x: x.type.lower() == "folder", items) # we should also de-duplicate before calculating folderItems = list(set(folderItems)) folders = len(folderItems) if 0 < limit < folders: # let's filter them by alphabet if the number is exceeded Logger.Debug("Creating Groups for list exceeding '%s' folder items. Total folders found '%s'.", limit, folders) other = LanguageHelper.GetLocalizedString(LanguageHelper.OtherChars) titleFormat = LanguageHelper.GetLocalizedString(LanguageHelper.StartWith) result = dict() nonGrouped = [] # prefixes = ("de", "het", "the", "een", "a", "an") for subItem in items: if subItem.dontGroup or subItem.type != "folder": nonGrouped.append(subItem) continue char = subItem.name[0].upper() # Should we de-prefix? # for p in prefixes: # if subItem.name.lower().startswith(p + " "): # char = subItem.name[len(p) + 1][0].upper() if char.isdigit(): char = "0-9" elif not char.isalpha(): char = other if char not in result: Logger.Trace("Creating Grouped item from: %s", subItem) if char == other: item = mediaitem.MediaItem(titleFormat.replace("'", "") % (char,), "") else: item = mediaitem.MediaItem(titleFormat % (char.upper(),), "") item.thumb = self.noImage item.complete = True # item.SetDate(2100 + ord(char[0]), 1, 1, text='') result[char] = item else: item = result[char] item.items.append(subItem) items = nonGrouped + result.values() Logger.Trace("Found '%s' items", len(items)) return list(set(items))
def UpdateVideoItem(self, item): """Updates an existing MediaItem with more data. Arguments: item : MediaItem - the MediaItem that needs to be updated Returns: The original item with more data added to it's properties. Used to update none complete MediaItems (self.complete = False). This could include opening the item's URL to fetch more data and then process that data or retrieve it's real media-URL. The method should at least: * cache the thumbnail to disk (use self.noImage if no thumb is available). * set at least one MediaItemPart with a single MediaStream. * set self.complete = True. if the returned item does not have a MediaItemPart then the self.complete flag will automatically be set back to False. """ Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name, self.channelName) data = UriHandler.Open(item.url, proxy=self.proxy, additionalHeaders=item.HttpHeaders) videoId = Regexer.DoRegex('data-video="([^"]+)"', data)[-1] url = "https://mediazone.vrt.be/api/v1/canvas/assets/%s" % (videoId, ) data = UriHandler.Open(url, proxy=self.proxy, additionalHeaders=item.HttpHeaders) json = JsonHelper(data) geoLocked = str(json.GetValue("metaInfo", "allowedRegion").lower()) hideGeoLocked = AddonSettings.HideGeoLockedItemsForLocation(geoLocked) if hideGeoLocked: geoRegion = AddonSettings.HideGeoLockedItemsForLocation( geoLocked, True) Logger.Warning( "Found GEO Locked item for region '%s'. Current region is '%s'", geoLocked, geoRegion) return item part = item.CreateNewEmptyMediaPart() for video in json.GetValue("targetUrls"): videoType = video["type"].lower() url = video["url"] if videoType == "progressive_download": bitrate = 1000 elif videoType == "hls": for s, b in M3u8.GetStreamsFromM3u8(url, self.proxy): # s = self.GetVerifiableVideoUrl(s) part.AppendMediaStream(s, b) continue elif videoType == "rtmp": # url=rtmp://vod.stream.vrt.be/mediazone_canvas/_definst_/mp4:2015/11/mz-ast-79a551d6-2621-4a0f-9af0-a272fb0954db-1/video_1296.mp4 url = url.replace("_definst_/mp4:", "?slist=") bitrate = 1100 else: Logger.Debug("Found unhandled stream type '%s':%s", videoType, url) continue part.AppendMediaStream(url, bitrate) item.complete = True return item