def ProcessFolderList(self, item=None):
"""Process the selected item and get's it's child items using the available dataparsers.
Arguments:
item : [opt] MediaItem - the selected item
Returns:
A list of MediaItems that form the childeren of the <item>.
Accepts an <item> and returns a list of MediaListems with at least name & url
set. The following actions are done:
* determining the correct parsers to use
* call a pre-processor
* parsing the data with the parsers
* calling the creators for item creations
if the item is NOne, we assume that we are dealing with the first call for this channel and the mainlist uri
is used.
"""
items = []
self.parentItem = item
if item is None:
Logger.Info("ProcessFolderList :: No item was specified. Assuming it was the main channel list")
url = self.mainListUri
headers = self.httpHeaders
elif len(item.items) > 0:
return item.items
else:
url = item.url
headers = item.HttpHeaders
if url.startswith("http:") or url.startswith("https:") or url.startswith("file:"):
data = UriHandler.Open(url, proxy=self.proxy, additionalHeaders=headers)
elif url.startswith("#"):
data = ""
elif url == "searchSite":
Logger.Debug("Starting to search")
return self.SearchSite()
else:
Logger.Debug("Unknown URL format. Setting data to ''")
data = ""
# Determine the handlers and process
dataParsers = self.__GetDataParsers(url)
# first check if there is a generic pre-processor
preProcs = filter(lambda p: p.IsGenericPreProcessor(), dataParsers)
numPreProcs = len(preProcs)
Logger.Trace("Processing %s Generic Pre-Processors DataParsers", numPreProcs)
if numPreProcs > 1:
# warn for strange results if more than 1 generic pre-processor is present.
Logger.Warning("More than one Generic Pre-Processor is found (%s). They are being processed in the "
"order that Python likes which might result in unexpected result.", numPreProcs)
for dataParser in preProcs:
# remove it from the list
dataParsers.remove(dataParser)
# and process it
Logger.Debug("Processing %s", dataParser)
(data, preItems) = dataParser.PreProcessor(data)
items += preItems
# The the other handlers
Logger.Trace("Processing %s Normal DataParsers", len(dataParsers))
handlerJson = None
for dataParser in dataParsers:
Logger.Debug("Processing %s", dataParser)
# Check for preprocessors
if dataParser.PreProcessor:
Logger.Debug("Processing DataParser.PreProcessor")
(handlerData, preItems) = dataParser.PreProcessor(data)
items += preItems
else:
handlerData = data
Logger.Debug("Processing DataParser.Parser")
if dataParser.Parser is None or (dataParser.Parser == "" and not dataParser.IsJson):
if dataParser.Creator:
Logger.Warning("No <parser> found for %s. Skipping.", dataParser.Creator)
continue
if dataParser.IsJson:
if handlerJson is None:
# Cache the json requests to improve performance
Logger.Trace("Caching JSON results for Dataparsing")
handlerJson = JsonHelper(handlerData, Logger.Instance())
Logger.Trace(dataParser.Parser)
parserResults = handlerJson.GetValue(fallback=[], *dataParser.Parser)
if not isinstance(parserResults, (tuple, list)):
# if there is just one match, return that as a list
parserResults = [parserResults]
else:
parserResults = Regexer.DoRegex(dataParser.Parser, handlerData)
Logger.Debug("Processing DataParser.Creator for %s items", len(parserResults))
for parserResult in parserResults:
handlerItem = dataParser.Creator(parserResult)
if handlerItem is not None:
items.append(handlerItem)
# should we exclude DRM/GEO?
hideGeoLocked = AddonSettings.HideGeoLockedItemsForLocation(self.language)
hideDrmProtected = AddonSettings.HideDrmItems()
hidePremium = AddonSettings.HidePremiumItems()
hideFolders = AddonSettings.HideRestrictedFolders()
typeToExclude = None
if not hideFolders:
typeToExclude = "folder"
oldCount = len(items)
if hideDrmProtected:
Logger.Debug("Hiding DRM items")
items = filter(lambda i: not i.isDrmProtected or i.type == typeToExclude, items)
if hideGeoLocked:
Logger.Debug("Hiding GEO Locked items due to GEO region: %s", self.language)
items = filter(lambda i: not i.isGeoLocked or i.type == typeToExclude, items)
if hidePremium:
Logger.Debug("Hiding Premium items")
items = filter(lambda i: not i.isPaid or i.type == typeToExclude, items)
# items = filter(lambda i: not i.isPaid or i.type == "folder", items)
if len(items) != oldCount:
Logger.Info("Hidden %s items due to DRM/GEO/Premium filter (Hide Folders=%s)",
oldCount - len(items), hideFolders)
# Check for grouping or not
limit = AddonSettings.GetListLimit()
folderItems = filter(lambda x: x.type.lower() == "folder", items)
# we should also de-duplicate before calculating
folderItems = list(set(folderItems))
folders = len(folderItems)
if 0 < limit < folders:
# let's filter them by alphabet if the number is exceeded
Logger.Debug("Creating Groups for list exceeding '%s' folder items. Total folders found '%s'.",
limit, folders)
other = LanguageHelper.GetLocalizedString(LanguageHelper.OtherChars)
titleFormat = LanguageHelper.GetLocalizedString(LanguageHelper.StartWith)
result = dict()
nonGrouped = []
# prefixes = ("de", "het", "the", "een", "a", "an")
for subItem in items:
if subItem.dontGroup or subItem.type != "folder":
nonGrouped.append(subItem)
continue
char = subItem.name[0].upper()
# Should we de-prefix?
# for p in prefixes:
# if subItem.name.lower().startswith(p + " "):
# char = subItem.name[len(p) + 1][0].upper()
if char.isdigit():
char = "0-9"
elif not char.isalpha():
char = other
if char not in result:
Logger.Trace("Creating Grouped item from: %s", subItem)
if char == other:
item = mediaitem.MediaItem(titleFormat.replace("'", "") % (char,), "")
else:
item = mediaitem.MediaItem(titleFormat % (char.upper(),), "")
item.thumb = self.noImage
item.complete = True
# item.SetDate(2100 + ord(char[0]), 1, 1, text='')
result[char] = item
else:
item = result[char]
item.items.append(subItem)
items = nonGrouped + result.values()
Logger.Trace("Found '%s' items", len(items))
return list(set(items))
def UpdateVideoItem(self, item):
"""Updates an existing MediaItem with more data.
Arguments:
item : MediaItem - the MediaItem that needs to be updated
Returns:
The original item with more data added to it's properties.
Used to update none complete MediaItems (self.complete = False). This
could include opening the item's URL to fetch more data and then process that
data or retrieve it's real media-URL.
The method should at least:
* cache the thumbnail to disk (use self.noImage if no thumb is available).
* set at least one MediaItemPart with a single MediaStream.
* set self.complete = True.
if the returned item does not have a MediaItemPart then the self.complete flag
will automatically be set back to False.
"""
Logger.Debug('Starting UpdateVideoItem for %s (%s)', item.name,
self.channelName)
data = UriHandler.Open(item.url,
proxy=self.proxy,
additionalHeaders=item.HttpHeaders)
videoId = Regexer.DoRegex('data-video="([^"]+)"', data)[-1]
url = "https://mediazone.vrt.be/api/v1/canvas/assets/%s" % (videoId, )
data = UriHandler.Open(url,
proxy=self.proxy,
additionalHeaders=item.HttpHeaders)
json = JsonHelper(data)
geoLocked = str(json.GetValue("metaInfo", "allowedRegion").lower())
hideGeoLocked = AddonSettings.HideGeoLockedItemsForLocation(geoLocked)
if hideGeoLocked:
geoRegion = AddonSettings.HideGeoLockedItemsForLocation(
geoLocked, True)
Logger.Warning(
"Found GEO Locked item for region '%s'. Current region is '%s'",
geoLocked, geoRegion)
return item
part = item.CreateNewEmptyMediaPart()
for video in json.GetValue("targetUrls"):
videoType = video["type"].lower()
url = video["url"]
if videoType == "progressive_download":
bitrate = 1000
elif videoType == "hls":
for s, b in M3u8.GetStreamsFromM3u8(url, self.proxy):
# s = self.GetVerifiableVideoUrl(s)
part.AppendMediaStream(s, b)
continue
elif videoType == "rtmp":
# url=rtmp://vod.stream.vrt.be/mediazone_canvas/_definst_/mp4:2015/11/mz-ast-79a551d6-2621-4a0f-9af0-a272fb0954db-1/video_1296.mp4
url = url.replace("_definst_/mp4:", "?slist=")
bitrate = 1100
else:
Logger.Debug("Found unhandled stream type '%s':%s", videoType,
url)
continue
part.AppendMediaStream(url, bitrate)
item.complete = True
return item
