def isImportReady(handle, options):
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log('cannot retrieve media import', xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log('cannot prepare media import settings', xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = xbmcmediaimport.getProvider(handle)
if not mediaProvider:
log('cannot retrieve media provider', xbmc.LOGERROR)
return
# prepare the media provider settings
if not mediaProvider.prepareSettings():
log('cannot prepare media provider settings', xbmc.LOGERROR)
return
try:
embyServer = Server(mediaProvider)
except:
return
# check if the chosen library views exist
selectedViews = ImportSettings.GetLibraryViews(importSettings)
matchingViews = getMatchingLibraryViews(embyServer,
mediaImport.getMediaTypes(),
selectedViews)
xbmcmediaimport.setImportReady(handle, len(matchingViews) > 0)
def execImport(handle, options):
if not 'path' in options:
log('cannot execute "import" without path', xbmc.LOGERROR)
return
# parse all necessary options
mediaTypes = mediaTypesFromOptions(options)
if not mediaTypes:
log('cannot execute "import" without media types', xbmc.LOGERROR)
return
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log('cannot retrieve media import', xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log('cannot prepare media import settings', xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = mediaImport.getProvider()
if not mediaProvider:
log('cannot retrieve media provider', xbmc.LOGERROR)
return
log('importing {} items from {}...'.format(
mediaTypes, mediaProvider2str(mediaProvider)))
# prepare the media provider settings
mediaProviderSettings = mediaProvider.prepareSettings()
if not mediaProviderSettings:
log('cannot prepare media provider settings', xbmc.LOGERROR)
return
# create an Emby server instance
embyServer = Server(mediaProvider)
# build the base URL to retrieve items
baseUrl = embyServer.BuildUserUrl(emby.constants.URL_ITEMS)
baseUrlOptions = {
emby.constants.URL_QUERY_ITEMS_RECURSIVE: 'true',
emby.constants.URL_QUERY_ITEMS_FIELDS: ','.join(EMBY_ITEM_FIELDS),
emby.constants.URL_QUERY_ITEMS_EXCLUDE_LOCATION_TYPES:
'Virtual,Offline',
emby.constants.URL_QUERY_ITEMS_LIMIT: ITEM_REQUEST_LIMIT
}
baseUrl = Url.addOptions(baseUrl, baseUrlOptions)
# get all (matching) library views
selectedViews = ImportSettings.GetLibraryViews(importSettings)
views = getMatchingLibraryViews(embyServer, mediaTypes, selectedViews)
if not views:
log('cannot retrieve items without any library views', xbmc.LOGERROR)
return
# determine whether Direct Play is allowed
allowDirectPlay = mediaProviderSettings.getBool(
emby.constants.SETTING_PROVIDER_PLAYBACK_ALLOW_DIRECT_PLAY)
# determine whether to import collections
importCollections = importSettings.getBool(
emby.constants.SETTING_IMPORT_IMPORT_COLLECTIONS)
# determine the last sync time and whether we should perform a fast sync
fastSync = False
syncUrlOptions = {}
# check if synchronization related settings have changed; if yes we have to perform a full synchronization
if SynchronizationSettings.HaveChanged(mediaTypes,
mediaProviderSettings,
importSettings,
save=True):
log('forcing a full synchronization to import {} items from {} because some related settings have changed' \
.format(mediaTypes, mediaProvider2str(mediaProvider)))
else:
# check if we
# have already performed a (full) synchronization before
# should use the Kodi Companion Emby server plugin
lastSync = mediaImport.getLastSynced()
if lastSync and mediaProviderSettings.getBool(
emby.constants.
SETTING_PROVIDER_SYNCHRONIZATION_USE_KODI_COMPANION):
if KodiCompanion.IsInstalled(embyServer):
fastSync = True
# convert the last sync datetime string to ISO 8601
lastSync = parser.parse(lastSync).astimezone(utc).isoformat(
timespec='seconds')
syncUrlOptions.update({
# only set MinDateLastSavedForUser because it already covers DateLastSaved, RatingLastModified
# and PlaystateLastModified. Setting both MinDateLastSaved and MinDateLastSavedForUser will
# cause issues, see https://emby.media/community/index.php?/topic/82258-retrieving-changeset-when-client-returns-online-mediaimport/
emby.constants.URL_QUERY_ITEMS_MIN_DATE_LAST_SAVED_FOR_USER:
lastSync
})
log('using fast synchronization to import {} items from {} with Kodi companion plugin' \
.format(mediaTypes, mediaProvider2str(mediaProvider)), xbmc.LOGDEBUG)
# retrieving the sync queue from Kodi companion
syncQueue = KodiCompanion.SyncQueue.GetItems(
embyServer, lastSync)
else:
log('Kodi companion usage is enabled to import {} items from {} but the server plugin is not installed' \
.format(mediaTypes, mediaProvider2str(mediaProvider)), xbmc.LOGWARNING)
# loop over all media types to be imported
progress = 0
progressTotal = len(mediaTypes)
for mediaType in mediaTypes:
if xbmcmediaimport.shouldCancel(handle, progress, progressTotal):
return
progress += 1
if mediaType == xbmcmediaimport.MediaTypeVideoCollection and not importCollections:
log(
'importing {} items from {} is disabled'.format(
mediaType, mediaProvider2str(mediaProvider)),
xbmc.LOGDEBUG)
continue
log('importing {} items from {}...'.format(
mediaType, mediaProvider2str(mediaProvider)))
mappedMediaType = kodi.Api.getEmbyMediaType(mediaType)
if not mappedMediaType:
log('cannot import unsupported media type "{}"'.format(mediaType),
xbmc.LOGERROR)
continue
(_, embyMediaType, localizedMediaType) = mappedMediaType
xbmcmediaimport.setProgressStatus(
handle,
__addon__.getLocalizedString(32001).format(
__addon__.getLocalizedString(localizedMediaType)))
urlOptions = syncUrlOptions.copy()
urlOptions.update(
{emby.constants.URL_QUERY_ITEMS_INCLUDE_ITEM_TYPES: embyMediaType})
url = Url.addOptions(baseUrl, urlOptions)
boxsetUrlOptions = {
emby.constants.URL_QUERY_ITEMS_INCLUDE_ITEM_TYPES:
kodi.EMBY_MEDIATYPE_BOXSET
}
boxsetUrl = Url.addOptions(baseUrl, boxsetUrlOptions)
items = []
boxsets = {}
# handle library views
for view in views:
log('importing {} items from "{}" view from {}...'.format(
mediaType, view.name, mediaProvider2str(mediaProvider)))
items.extend(
importItems(handle,
embyServer,
url,
mediaType,
view.id,
embyMediaType=embyMediaType,
viewName=view.name,
allowDirectPlay=allowDirectPlay))
if importCollections and items and mediaType == xbmcmediaimport.MediaTypeMovie:
# retrieve all BoxSets / collections matching the current media type
boxsetObjs = importItems(handle,
embyServer,
boxsetUrl,
mediaType,
view.id,
raw=True,
allowDirectPlay=allowDirectPlay)
for boxsetObj in boxsetObjs:
if not emby.constants.PROPERTY_ITEM_ID in boxsetObj or not emby.constants.PROPERTY_ITEM_NAME in boxsetObj:
continue
boxsetId = boxsetObj[emby.constants.PROPERTY_ITEM_ID]
boxsetName = boxsetObj[emby.constants.PROPERTY_ITEM_NAME]
boxsets[boxsetId] = boxsetName
# handle BoxSets / collections
if importCollections and items:
for (boxsetId, boxsetName) in iteritems(boxsets):
# get all items belonging to the BoxSet
boxsetItems = importItems(handle,
embyServer,
url,
mediaType,
boxsetId,
embyMediaType=embyMediaType,
viewName=boxsetName,
allowDirectPlay=allowDirectPlay)
for boxsetItem in boxsetItems:
# find the matching retrieved item
for index, item in enumerate(items):
if boxsetItem.getPath() == item.getPath():
# set the BoxSet / collection
kodi.Api.setCollection(item, boxsetName)
items[index] = item
# in a fast sync we need to get the removed items from Kodi companion
if fastSync:
if items:
log('{} changed {} items imported from {}'.format(
len(items), mediaType, mediaProvider2str(mediaProvider)))
# handle removed items through Kodi companion's sync queue
if syncQueue.itemsRemoved:
# retrieve all local items matching the current media type from the current import
localItems = xbmcmediaimport.getImportedItems(
handle, mediaType)
# match the local items against the changed items
removedItems, = kodi.Api.matchImportedItemIdsToLocalItems(
localItems, syncQueue.itemsRemoved) # pylint: disable=unbalanced-tuple-unpacking
# erase all removed items matching the current media type from the sync queue
syncQueue.itemsRemoved = [
removedItem for removedItem in syncQueue.itemsRemoved
if removedItem in removedItems
]
if removedItems:
log('{} previously imported {} items removed from {}'.
format(len(removedItems), mediaType,
mediaProvider2str(mediaProvider)))
xbmcmediaimport.addImportItems(
handle, removedItems, mediaType,
xbmcmediaimport.MediaImportChangesetTypeRemoved)
else:
log('{} {} items imported from {}'.format(
len(items), mediaType, mediaProvider2str(mediaProvider)))
# pass the imported items back to Kodi
if items:
xbmcmediaimport.addImportItems(handle, items, mediaType)
xbmcmediaimport.finishImport(handle, fastSync)
def run(self):
numRetriesOnTimeout = ImportSettings.GetNumberOfRetriesOnTimeout(
self._media_import)
numSecondsBetweenRetries = ImportSettings.GetNumberOfSecondsBetweenRetries(
self._media_import)
while not self.should_stop():
while not self.should_stop():
plex_item = None
try:
plex_item = self._items_to_process_queue.get_nowait()
except Empty:
# if the queue is empty and we should finish, return completely
if self.should_finish():
return
break
converted_item = None
retries = numRetriesOnTimeout
while retries > 0:
try:
# manually reload the item's metadata
if isinstance(plex_item, PlexPartialObject
) and not plex_item.isFullObject():
plex_item.reload(
**ToFileItemConverterThread.INCLUDES)
# convert the plex item to a ListItem
converted_item = Api.toFileItem(
self._plex_server,
plex_item,
mediaType=self._media_type,
plexLibType=self._plex_lib_type,
allowDirectPlay=self._allow_direct_play)
# get out of the retry loop
break
except Exception as e:
# Api.convertDateTimeToDbDateTime may return (404) not_found for orphaned items in the library
log((
f"failed to retrieve item {plex_item.title} with key {plex_item.key} "
f"from {mediaProvider2str(self._media_provider)}: {e}"
), LOGWARNING)
# retry after timeout
retries -= 1
# check if there are any more retries left
# if not skip the item
if retries == 0:
log((
f"retrieving item {plex_item.title} with key {plex_item.key} from "
f"{mediaProvider2str(self._media_provider)} failed after "
f"{numRetriesOnTimeout} retries"), LOGWARNING)
else:
# otherwise wait before trying again
log((
f"retrying to retrieve {plex_item.title} with key {plex_item.key} from "
f"{mediaProvider2str(self._media_provider)} in "
f"{numSecondsBetweenRetries} seconds"))
sleep(float(numSecondsBetweenRetries))
# let the input queue know that the plex item has been processed
self._items_to_process_queue.task_done()
if converted_item:
# put the converted item into the output queue
self._processed_items_queue.put(converted_item)
else:
log((
f"failed to convert item {plex_item.title} with key {plex_item.key} "
f"from {mediaProvider2str(self._media_provider)}"),
LOGWARNING)
self._count_items_to_process -= 1
self._count_processed_items += 1
# wait for the stop event
self._stop_event.wait(0.1)
def execImport(handle: int, options: dict):
"""Perform library update/import of all configured items from a configured PMS into Kodi
:param handle: Handle id from input
:type handle: int
:param options: Options/parameters passed in with the call, required mediatypes or mediatypes[]
:type options: dict
"""
# parse all necessary options
mediaTypes = mediaTypesFromOptions(options)
if not mediaTypes:
log("cannot execute 'import' without media types", xbmc.LOGERROR)
return
# retrieve the media import
mediaImport = xbmcmediaimport.getImport(handle)
if not mediaImport:
log("cannot retrieve media import", xbmc.LOGERROR)
return
# prepare and get the media import settings
importSettings = mediaImport.prepareSettings()
if not importSettings:
log("cannot prepare media import settings", xbmc.LOGERROR)
return
# retrieve the media provider
mediaProvider = mediaImport.getProvider()
if not mediaProvider:
log("cannot retrieve media provider", xbmc.LOGERROR)
return
# prepare and get the media provider settings
providerSettings = mediaProvider.prepareSettings()
if not providerSettings:
log("cannot prepare provider settings", xbmc.LOGERROR)
return
# get direct play settings from provider
allowDirectPlay = providerSettings.getBool(SETTINGS_PROVIDER_PLAYBACK_ALLOW_DIRECT_PLAY)
# create a Plex Media Server instance
server = Server(mediaProvider)
plexServer = server.PlexServer()
plexLibrary = plexServer.library
# get all (matching) library sections
selectedLibrarySections = getLibrarySectionsFromSettings(importSettings)
librarySections = getMatchingLibrarySections(plexServer, mediaTypes, selectedLibrarySections)
if not librarySections:
log(f"cannot retrieve {mediaTypes} items without any library section", xbmc.LOGERROR)
return
numDownloadThreads = ImportSettings.GetNumberOfDownloadThreads(importSettings)
numRetriesOnTimeout = ImportSettings.GetNumberOfRetriesOnTimeout(importSettings)
numSecondsBetweenRetries = ImportSettings.GetNumberOfSecondsBetweenRetries(importSettings)
# Decide if doing fast sync or not, if so set filter string to include updatedAt
fastSync = True
lastSync = mediaImport.getLastSynced()
# Check if import settings have changed, or if this is the first time we are importing this library type
if not lastSync:
fastSync = False
SynchronizationSettings.CalculateHash(
importSettings=importSettings,
providerSettings=providerSettings,
save=True
)
log("first time syncronizing library, forcing a full syncronization", xbmc.LOGINFO)
elif SynchronizationSettings.HaveChanged(
importSettings=importSettings,
providerSettings=providerSettings,
save=True
):
fastSync = False
log("library import settings have changed, forcing a full syncronization", xbmc.LOGINFO)
if SynchronizationSettings.HaveChanged(importSettings=importSettings, providerSettings=providerSettings, save=True):
fastSync = False
log("library import settings have changed, forcing a full syncronization", xbmc.LOGINFO)
if fastSync:
log(f"performing fast syncronization of items viewed or updated since {str(lastSync)}")
# loop over all media types to be imported
progressTotal = len(mediaTypes)
for progress, mediaType in enumerate(mediaTypes):
if xbmcmediaimport.shouldCancel(handle, progress, progressTotal):
return
mappedMediaType = Api.getPlexMediaType(mediaType)
if not mappedMediaType:
log(f"cannot import unsupported media type '{mediaType}'", xbmc.LOGERROR)
continue
plexLibType = mappedMediaType['libtype']
localizedMediaType = localize(mappedMediaType['label']).decode()
# prepare and start the converter threads
converterThreads = []
for _ in range(0, numDownloadThreads):
converterThreads.append(ToFileItemConverterThread(mediaProvider, mediaImport, plexServer,
media_type=mediaType, plex_lib_type=plexLibType, allow_direct_play=allowDirectPlay))
log((
f"starting {len(converterThreads)} threads to import {mediaType} items "
f"from {mediaProvider2str(mediaProvider)}..."),
xbmc.LOGDEBUG)
for converterThread in converterThreads:
converterThread.start()
# prepare function to stop all converter threads
def stopConverterThreads(converterThreads):
log((
f"stopping {len(converterThreads)} threads importing {mediaType} items "
f"from {mediaProvider2str(mediaProvider)}..."),
xbmc.LOGDEBUG)
for converterThread in converterThreads:
converterThread.stop()
xbmcmediaimport.setProgressStatus(handle, localize(32001, localizedMediaType))
log(f"importing {mediaType} items from {mediaProvider2str(mediaProvider)}", xbmc.LOGINFO)
# handle library sections
sectionsProgressTotal = len(librarySections)
for sectionsProgress, librarySection in enumerate(librarySections):
if xbmcmediaimport.shouldCancel(handle, sectionsProgress, sectionsProgressTotal):
stopConverterThreads(converterThreads)
return
# get the library section from the Plex Media Server
section = plexLibrary.sectionByID(librarySection['key'])
if not section:
log(f"cannot import {mediaType} items from unknown library section {librarySection}", xbmc.LOGWARNING)
continue
# get all matching items from the library section and turn them into ListItems
sectionRetrievalProgress = 0
sectionConversionProgress = 0
sectionProgressTotal = ITEM_REQUEST_LIMIT
while sectionRetrievalProgress < sectionProgressTotal:
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
maxResults = min(ITEM_REQUEST_LIMIT, sectionProgressTotal - sectionRetrievalProgress)
retries = numRetriesOnTimeout
while retries > 0:
try:
if fastSync:
lastSyncDatetime = parser.parse(lastSync).astimezone(timezone.utc)
prefix = ''
if mediaType in (xbmcmediaimport.MediaTypeTvShow, xbmcmediaimport.MediaTypeEpisode):
prefix = Api.getPlexMediaType(mediaType)['libtype'] + '.'
elif mediaType == xbmcmediaimport.MediaTypeSeason:
prefix = Api.getPlexMediaType(xbmcmediaimport.MediaTypeEpisode)['libtype'] + '.'
updatedFilter = {prefix + 'updatedAt>>': lastSyncDatetime}
watchedFilter = {prefix + 'lastViewedAt>>': lastSyncDatetime}
updatedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
filters=updatedFilter
)
log(f"discovered {len(updatedPlexItems)} updated {mediaType} items from {mediaProvider2str(mediaProvider)}")
watchedPlexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
filters=watchedFilter
)
log(f"discovered {len(watchedPlexItems)} newly watched {mediaType} items from {mediaProvider2str(mediaProvider)}")
plexItems = updatedPlexItems
plexItems.extend(
[item for item in watchedPlexItems if item.key not in [item.key for item in plexItems]]
)
else:
plexItems = section.search(
libtype=plexLibType,
container_start=sectionRetrievalProgress,
container_size=maxResults,
maxresults=maxResults,
)
# get out of the retry loop
break
except Exception as e:
log(f"failed to fetch {mediaType} items from {mediaProvider2str(mediaProvider)}: {e}", xbmc.LOGWARNING)
# retry after timeout
retries -= 1
# check if there are any more retries left
# if not abort the import process
if retries == 0:
log(
(
f"fetching {mediaType} items from {mediaProvider2str(mediaProvider)} failed "
f"after {numRetriesOnTimeout} retries"
),
xbmc.LOGWARNING)
stopConverterThreads(converterThreads)
return
else:
# otherwise wait before trying again
log(
(
f"retrying to fetch {mediaType} items from {mediaProvider2str(mediaProvider)} in "
f"{numSecondsBetweenRetries} seconds"
)
)
time.sleep(float(numSecondsBetweenRetries))
# Update sectionProgressTotal now that search has run and totalSize has been updated
# TODO(Montellese): fix access of private LibrarySection._totalViewSize
sectionProgressTotal = section._totalViewSize
# nothing to do if no items have been retrieved from Plex
if not plexItems:
continue
# automatically determine how to distribute the retrieved items across the available converter threads
plexItemsPerConverter, remainingPlexItems = divmod(len(plexItems), len(converterThreads))
plexItemsPerConverters = [plexItemsPerConverter] * len(converterThreads)
plexItemsPerConverters[0:remainingPlexItems] = [plexItemsPerConverter + 1] * remainingPlexItems
converterThreadIndex = 0
splitItems = []
for plexItem in plexItems:
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
sectionRetrievalProgress += 1
# copllect the plex items for the next converter thread
splitItems.append(plexItem)
plexItemsPerConverters[converterThreadIndex] -= 1
# move to the next converter thread if necessary
if not plexItemsPerConverters[converterThreadIndex]:
converterThreads[converterThreadIndex].add_items_to_convert(splitItems)
splitItems.clear()
converterThreadIndex += 1
# retrieve and combine the progress of all converter threads
sectionConversionProgress = \
sum(converterThread.get_converted_items_count() for converterThread in converterThreads)
if splitItems:
log(f"forgot to process {len(splitItems)} {mediaType} items", xbmc.LOGWARNING)
# retrieve converted items from the converter threads
totalItemsToImport = 0
countFinishedConverterThreads = 0
while countFinishedConverterThreads < len(converterThreads):
if xbmcmediaimport.shouldCancel(handle, sectionConversionProgress, sectionProgressTotal):
stopConverterThreads(converterThreads)
return
sectionConversionProgress = 0
itemsToImport = []
for converterThread in converterThreads:
# update the progress
sectionConversionProgress += converterThread.get_converted_items_count()
# ignore finished converter threads
if converterThread.should_finish():
continue
# retrieve the converted items
convertedItems = converterThread.get_converted_items()
itemsToImport.extend(convertedItems)
# check if all items have been converted
if not converterThread.get_items_to_convert_count():
converterThread.finish()
countFinishedConverterThreads += 1
if itemsToImport:
totalItemsToImport += len(itemsToImport)
xbmcmediaimport.addImportItems(handle, itemsToImport, mediaType)
time.sleep(0.1)
if totalItemsToImport:
log(f"{totalItemsToImport} {mediaType} items imported from {mediaProvider2str(mediaProvider)}", xbmc.LOGINFO)
xbmcmediaimport.finishImport(handle, fastSync)