def execImport(handle: int, options: dict):
"""Perform library update/import of all configured items from a configured PMS into Kodi
:param handle: Handle id from input
:type handle: int
:param options: Options/parameters passed in with the call, required mediatypes or mediatypes[]
:type options: dict
"""
if 'path' not in options:
log("cannot execute 'import' without path", xbmc.LOGERROR)
return
# parse all necessary options
mediaTypes = mediaTypesFromOptions(options)
if not mediaTypes:
log("cannot execute 'import' without media types", xbmc.LOGERROR)
return
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log("cannot retrieve media import", xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log("cannot prepare media import settings", xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = mediaImport.getProvider()
if not mediaProvider:
log("cannot retrieve media provider", xbmc.LOGERROR)
return
# prepare and get the media provider settings
providerSettings = mediaProvider.prepareSettings()
if not providerSettings:
log("cannot prepare provider settings", xbmc.LOGERROR)
return
# create a Plex Media Server instance
server = Server(mediaProvider)
plexServer = server.PlexServer()
plexLibrary = plexServer.library
# get all (matching) library sections
selectedLibrarySections = getLibrarySectionsFromSettings(importSettings)
librarySections = getMatchingLibrarySections(plexServer, mediaTypes,
selectedLibrarySections)
if not librarySections:
log(f"cannot retrieve {mediaTypes} items without any library section",
xbmc.LOGERROR)
return
# Decide if doing fast sync or not, if so set filter string to include updatedAt
fastSync = True
lastSync = mediaImport.getLastSynced()
# Check if import settings have changed, or if this is the first time we are importing this library type
if not lastSync:
fastSync = False
SynchronizationSettings.CalculateHash(
importSettings=importSettings,
providerSettings=providerSettings,
save=True)
log("first time syncronizing library, forcing a full syncronization",
xbmc.LOGINFO)
elif SynchronizationSettings.HaveChanged(importSettings=importSettings,
providerSettings=providerSettings,
save=True):
fastSync = False
log(
"library import settings have changed, forcing a full syncronization",
xbmc.LOGINFO)
if SynchronizationSettings.HaveChanged(importSettings=importSettings,
providerSettings=providerSettings,
save=True):
fastSync = False
log(
"library import settings have changed, forcing a full syncronization",
xbmc.LOGINFO)
if fastSync:
log(f"performing fast syncronization of items viewed or updated since {str(lastSync)}"
)
lastSyncEpoch = parser.parse(lastSync).strftime('%s')
updatedFilter = {'updatedAt>': lastSyncEpoch}
watchedFilter = {'lastViewedAt>': lastSyncEpoch}
# loop over all media types to be imported
progressTotal = len(mediaTypes)
for progress, mediaType in enumerate(mediaTypes):
if xbmcmediaimport.shouldCancel(handle, progress, progressTotal):
return
mappedMediaType = Api.getPlexMediaType(mediaType)
if not mappedMediaType:
log(f"cannot import unsupported media type '{mediaType}'",
xbmc.LOGERROR)
continue
plexLibType = mappedMediaType['libtype']
localizedMediaType = localize(mappedMediaType['label']).decode()
xbmcmediaimport.setProgressStatus(handle,
localize(32001, localizedMediaType))
log(
f"importing {mediaType} items from {mediaProvider2str(mediaProvider)}",
xbmc.LOGINFO)
# handle library sections
itemsToImport = []
sectionsProgressTotal = len(librarySections)
for sectionsProgress, librarySection in enumerate(librarySections):
if xbmcmediaimport.shouldCancel(handle, sectionsProgress,
sectionsProgressTotal):
return
# get the library section from the Plex Media Server
section = plexLibrary.sectionByID(librarySection['key'])
if not section:
log(
f"cannot import {mediaType} items from unknown library section {librarySection}",
xbmc.LOGWARNING)
continue
# get all matching items from the library section and turn them into ListItems
sectionProgress = 0
sectionProgressTotal = ITEM_REQUEST_LIMIT
while sectionProgress < sectionProgressTotal:
if xbmcmediaimport.shouldCancel(handle, sectionProgress,
sectionProgressTotal):
return
maxResults = min(ITEM_REQUEST_LIMIT,
sectionProgressTotal - sectionProgress)
try:
if fastSync:
updatedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionProgress,
container_size=maxResults,
maxresults=maxResults,
**updatedFilter)
log(f"discovered {len(updatedPlexItems)} updated items from {mediaProvider2str(mediaProvider)}"
)
watchedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionProgress,
container_size=maxResults,
maxresults=maxResults,
**watchedFilter)
log(f"discovered {len(watchedPlexItems)} new watched items from {mediaProvider2str(mediaProvider)}"
)
plexItems = updatedPlexItems
plexItems.extend([
item for item in watchedPlexItems if item.key
not in [item.key for item in plexItems]
])
else:
plexItems = section.search(
libtype=plexLibType,
container_start=sectionProgress,
container_size=maxResults,
maxresults=maxResults,
)
except plexapi.exceptions.BadRequest as e:
log(
f"failed to fetch {mediaType} items from {mediaProvider2str(mediaProvider)}: {e}",
xbmc.LOGINFO)
return
# Update sectionProgressTotal now that search has run and totalSize has been updated
sectionProgressTotal = section.totalSize
plexItemsProgressTotal = len(plexItems)
for plexItemsProgress, plexItem in enumerate(plexItems):
if xbmcmediaimport.shouldCancel(handle, plexItemsProgress,
plexItemsProgressTotal):
return
sectionProgress += 1
try:
item = Api.toFileItem(plexServer, plexItem, mediaType,
plexLibType)
if not item:
continue
itemsToImport.append(item)
except plexapi.exceptions.BadRequest as e:
# Api.convertDateTimeToDbDateTime may return (404) not_found for orphaned items in the library
log((
f"failed to retrieve item {plexItem.title} with key {plexItem.key} "
f"from {mediaProvider2str(mediaProvider)}: {e}"),
xbmc.LOGWARNING)
continue
if itemsToImport:
log(
f"{len(itemsToImport)} {mediaType} items imported from {mediaProvider2str(mediaProvider)}",
xbmc.LOGINFO)
xbmcmediaimport.addImportItems(handle, itemsToImport, mediaType)
xbmcmediaimport.finishImport(handle, fastSync)
def execImport(handle, options):
if not 'path' in options:
log('cannot execute "import" without path', xbmc.LOGERROR)
return
# parse all necessary options
mediaTypes = mediaTypesFromOptions(options)
if not mediaTypes:
log('cannot execute "import" without media types', xbmc.LOGERROR)
return
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log('cannot retrieve media import', xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log('cannot prepare media import settings', xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = mediaImport.getProvider()
if not mediaProvider:
log('cannot retrieve media provider', xbmc.LOGERROR)
return
# prepare the media provider settings
if not mediaProvider.prepareSettings():
log('cannot prepare media provider settings', xbmc.LOGERROR)
return
# create a Plex Media Server instance
server = Server(mediaProvider)
plexServer = server.PlexServer()
plexLibrary = plexServer.library
# get all (matching) library sections
selectedLibrarySections = getLibrarySectionsFromSettings(importSettings)
librarySections = getMatchingLibrarySections(plexServer, mediaTypes, selectedLibrarySections)
if not librarySections:
log('cannot retrieve {} items without any library section'.format(mediaTypes), xbmc.LOGERROR)
return
# loop over all media types to be imported
progressTotal = len(mediaTypes)
for progress, mediaType in enumerate(mediaTypes):
if xbmcmediaimport.shouldCancel(handle, progress, progressTotal):
return
mappedMediaType = Api.getPlexMediaType(mediaType)
if not mappedMediaType:
log('cannot import unsupported media type "{}"'.format(mediaType), xbmc.LOGERROR)
continue
plexLibType = mappedMediaType['libtype']
localizedMediaType = localise(mappedMediaType['label'])
xbmcmediaimport.setProgressStatus(handle, localise(32001).format(localizedMediaType))
log('importing {} items from {}'.format(mediaType, mediaProvider2str(mediaProvider)))
# handle library sections
plexItems = []
sectionsProgressTotal = len(librarySections)
for sectionsProgress, librarySection in enumerate(librarySections):
if xbmcmediaimport.shouldCancel(handle, sectionsProgress, sectionsProgressTotal):
return
# get the library section from the Plex Media Server
section = plexLibrary.sectionByID(librarySection['key'])
if not section:
log('cannot import {} items from unknown library section {}'.format(mediaType, librarySection), xbmc.LOGWARNING)
continue
# get all matching items from the library section
try:
plexSectionItems = section.search(libtype=plexLibType)
plexItems.extend(plexSectionItems)
except plexapi.exceptions.BadRequest as err:
log('failed to retrieve {} items from {}: {}'.format(mediaType, mediaProvider2str(mediaProvider), err))
return
# parse all items
items = []
itemsProgressTotal = len(plexItems)
for itemsProgress, plexItem in enumerate(plexItems):
if xbmcmediaimport.shouldCancel(handle, itemsProgress, itemsProgressTotal):
return
item = Api.toFileItem(plexServer, plexItem, mediaType, plexLibType)
if not item:
continue
items.append(item)
if items:
log('{} {} items imported from {}'.format(len(items), mediaType, mediaProvider2str(mediaProvider)))
xbmcmediaimport.addImportItems(handle, items, mediaType)
xbmcmediaimport.finishImport(handle)
def execImport(handle: int, options: dict):
"""Perform library update/import of all configured items from a configured PMS into Kodi
:param handle: Handle id from input
:type handle: int
:param options: Options/parameters passed in with the call, required mediatypes or mediatypes[]
:type options: dict
"""
# parse all necessary options
mediaTypes = mediaTypesFromOptions(options)
if not mediaTypes:
log("cannot execute 'import' without media types", xbmc.LOGERROR)
return
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log("cannot retrieve media import", xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log("cannot prepare media import settings", xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = mediaImport.getProvider()
if not mediaProvider:
log("cannot retrieve media provider", xbmc.LOGERROR)
return
# prepare and get the media provider settings
providerSettings = mediaProvider.prepareSettings()
if not providerSettings:
log("cannot prepare provider settings", xbmc.LOGERROR)
return
# get direct play settings from provider
allowDirectPlay = providerSettings.getBool(SETTINGS_PROVIDER_PLAYBACK_ALLOW_DIRECT_PLAY)
# create a Plex Media Server instance
server = Server(mediaProvider)
plexServer = server.PlexServer()
plexLibrary = plexServer.library
# get all (matching) library sections
selectedLibrarySections = getLibrarySectionsFromSettings(importSettings)
librarySections = getMatchingLibrarySections(plexServer, mediaTypes, selectedLibrarySections)
if not librarySections:
log(f"cannot retrieve {mediaTypes} items without any library section", xbmc.LOGERROR)
return
numDownloadThreads = ImportSettings.GetNumberOfDownloadThreads(importSettings)
numRetriesOnTimeout = ImportSettings.GetNumberOfRetriesOnTimeout(importSettings)
numSecondsBetweenRetries = ImportSettings.GetNumberOfSecondsBetweenRetries(importSettings)
# Decide if doing fast sync or not, if so set filter string to include updatedAt
fastSync = True
lastSync = mediaImport.getLastSynced()
# Check if import settings have changed, or if this is the first time we are importing this library type
if not lastSync:
fastSync = False
SynchronizationSettings.CalculateHash(
importSettings=importSettings,
providerSettings=providerSettings,
save=True
)
log("first time syncronizing library, forcing a full syncronization", xbmc.LOGINFO)
elif SynchronizationSettings.HaveChanged(
importSettings=importSettings,
providerSettings=providerSettings,
save=True
):
fastSync = False
log("library import settings have changed, forcing a full syncronization", xbmc.LOGINFO)
if SynchronizationSettings.HaveChanged(importSettings=importSettings, providerSettings=providerSettings, save=True):
fastSync = False
log("library import settings have changed, forcing a full syncronization", xbmc.LOGINFO)
if fastSync:
log(f"performing fast syncronization of items viewed or updated since {str(lastSync)}")
# loop over all media types to be imported
progressTotal = len(mediaTypes)
for progress, mediaType in enumerate(mediaTypes):
if xbmcmediaimport.shouldCancel(handle, progress, progressTotal):
return
mappedMediaType = Api.getPlexMediaType(mediaType)
if not mappedMediaType:
log(f"cannot import unsupported media type '{mediaType}'", xbmc.LOGERROR)
continue
plexLibType = mappedMediaType['libtype']
localizedMediaType = localize(mappedMediaType['label']).decode()
# prepare and start the converter threads
converterThreads = []
for _ in range(0, numDownloadThreads):
converterThreads.append(ToFileItemConverterThread(mediaProvider, mediaImport, plexServer,
media_type=mediaType, plex_lib_type=plexLibType, allow_direct_play=allowDirectPlay))
log((
f"starting {len(converterThreads)} threads to import {mediaType} items "
f"from {mediaProvider2str(mediaProvider)}..."),
xbmc.LOGDEBUG)
for converterThread in converterThreads:
converterThread.start()
# prepare function to stop all converter threads
def stopConverterThreads(converterThreads):
log((
f"stopping {len(converterThreads)} threads importing {mediaType} items "
f"from {mediaProvider2str(mediaProvider)}..."),
xbmc.LOGDEBUG)
for converterThread in converterThreads:
converterThread.stop()
xbmcmediaimport.setProgressStatus(handle, localize(32001, localizedMediaType))
log(f"importing {mediaType} items from {mediaProvider2str(mediaProvider)}", xbmc.LOGINFO)
# handle library sections
sectionsProgressTotal = len(librarySections)
for sectionsProgress, librarySection in enumerate(librarySections):
if xbmcmediaimport.shouldCancel(handle, sectionsProgress, sectionsProgressTotal):
stopConverterThreads(converterThreads)
return
# get the library section from the Plex Media Server
section = plexLibrary.sectionByID(librarySection['key'])
if not section:
log(f"cannot import {mediaType} items from unknown library section {librarySection}", xbmc.LOGWARNING)
continue
# get all matching items from the library section and turn them into ListItems
sectionRetrievalProgress = 0
sectionConversionProgress = 0
sectionProgressTotal = ITEM_REQUEST_LIMIT
while sectionRetrievalProgress < sectionProgressTotal:
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
maxResults = min(ITEM_REQUEST_LIMIT, sectionProgressTotal - sectionRetrievalProgress)
retries = numRetriesOnTimeout
while retries > 0:
try:
if fastSync:
lastSyncDatetime = parser.parse(lastSync).astimezone(timezone.utc)
prefix = ''
if mediaType in (xbmcmediaimport.MediaTypeTvShow, xbmcmediaimport.MediaTypeEpisode):
prefix = Api.getPlexMediaType(mediaType)['libtype'] + '.'
elif mediaType == xbmcmediaimport.MediaTypeSeason:
prefix = Api.getPlexMediaType(xbmcmediaimport.MediaTypeEpisode)['libtype'] + '.'
updatedFilter = {prefix + 'updatedAt>>': lastSyncDatetime}
watchedFilter = {prefix + 'lastViewedAt>>': lastSyncDatetime}
updatedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
filters=updatedFilter
)
log(f"discovered {len(updatedPlexItems)} updated {mediaType} items from {mediaProvider2str(mediaProvider)}")
watchedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
filters=watchedFilter
)
log(f"discovered {len(watchedPlexItems)} newly watched {mediaType} items from {mediaProvider2str(mediaProvider)}")
plexItems = updatedPlexItems
plexItems.extend(
[item for item in watchedPlexItems if item.key not in [item.key for item in plexItems]]
)
else:
plexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
)
# get out of the retry loop
break
except Exception as e:
log(f"failed to fetch {mediaType} items from {mediaProvider2str(mediaProvider)}: {e}", xbmc.LOGWARNING)
# retry after timeout
retries -= 1
# check if there are any more retries left
# if not abort the import process
if retries == 0:
log(
(
f"fetching {mediaType} items from {mediaProvider2str(mediaProvider)} failed "
f"after {numRetriesOnTimeout} retries"
),
xbmc.LOGWARNING)
stopConverterThreads(converterThreads)
return
else:
# otherwise wait before trying again
log(
(
f"retrying to fetch {mediaType} items from {mediaProvider2str(mediaProvider)} in "
f"{numSecondsBetweenRetries} seconds"
)
)
time.sleep(float(numSecondsBetweenRetries))
# Update sectionProgressTotal now that search has run and totalSize has been updated
# TODO(Montellese): fix access of private LibrarySection._totalViewSize
sectionProgressTotal = section._totalViewSize
# nothing to do if no items have been retrieved from Plex
if not plexItems:
continue
# automatically determine how to distribute the retrieved items across the available converter threads
plexItemsPerConverter, remainingPlexItems = divmod(len(plexItems), len(converterThreads))
plexItemsPerConverters = [plexItemsPerConverter] * len(converterThreads)
plexItemsPerConverters[0:remainingPlexItems] = [plexItemsPerConverter + 1] * remainingPlexItems
converterThreadIndex = 0
splitItems = []
for plexItem in plexItems:
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
sectionRetrievalProgress += 1
# copllect the plex items for the next converter thread
splitItems.append(plexItem)
plexItemsPerConverters[converterThreadIndex] -= 1
# move to the next converter thread if necessary
if not plexItemsPerConverters[converterThreadIndex]:
converterThreads[converterThreadIndex].add_items_to_convert(splitItems)
splitItems.clear()
converterThreadIndex += 1
# retrieve and combine the progress of all converter threads
sectionConversionProgress = \
sum(converterThread.get_converted_items_count() for converterThread in converterThreads)
if splitItems:
log(f"forgot to process {len(splitItems)} {mediaType} items", xbmc.LOGWARNING)
# retrieve converted items from the converter threads
totalItemsToImport = 0
countFinishedConverterThreads = 0
while countFinishedConverterThreads < len(converterThreads):
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
sectionConversionProgress = 0
itemsToImport = []
for converterThread in converterThreads:
# update the progress
sectionConversionProgress += converterThread.get_converted_items_count()
# ignore finished converter threads
if converterThread.should_finish():
continue
# retrieve the converted items
convertedItems = converterThread.get_converted_items()
itemsToImport.extend(convertedItems)
# check if all items have been converted
if not converterThread.get_items_to_convert_count():
converterThread.finish()
countFinishedConverterThreads += 1
if itemsToImport:
totalItemsToImport += len(itemsToImport)
xbmcmediaimport.addImportItems(handle, itemsToImport, mediaType)
time.sleep(0.1)
if totalItemsToImport:
log(f"{totalItemsToImport} {mediaType} items imported from {mediaProvider2str(mediaProvider)}", xbmc.LOGINFO)
xbmcmediaimport.finishImport(handle, fastSync)