def worker(self):
if not control.setting('metadata') == 'true': return
self.imdb_info_link = 'https://www.omdbapi.com/?i=%s&plot=full&r=json'
self.tvmaze_info_link = 'https://api.tvmaze.com/lookup/shows?thetvdb=%s'
self.lang = 'en'
self.meta = []
total = len(self.list)
if total == 0: return
for i in range(0, total):
self.list[i].update({'metacache': False})
self.list = metacache.fetch(self.list, self.lang)
multi = [i['imdb'] for i in self.list]
multi = [x for y,x in enumerate(multi) if x not in multi[:y]]
if len(multi) == 1:
self.movie_info(0) ; self.tv_info(0)
if self.meta: metacache.insert(self.meta)
for i in range(0, total): self.list[i].update({'metacache': False})
self.list = metacache.fetch(self.list, self.lang)
for r in range(0, total, 50):
threads = []
for i in range(r, r+50):
if i <= total: threads.append(workers.Thread(self.movie_info, i))
if i <= total: threads.append(workers.Thread(self.tv_info, i))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
metacache.insert(self.meta)
def tmdb_list(self, url):
try:
result = get_request(url % API_key)
items = result['results']
except: return
self.list = [] ; sortList = []
try:
page = int(result['page'])
total = int(result['total_pages'])
if page >= total: raise Exception()
if 'page=' not in url: raise Exception()
next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1)
except: next = ''
for item in items:
try:
values = {}
values['next'] = next
values['tmdb'] = str(item.get('id', '')) if item.get('id') else ''
sortList.append(values['tmdb'])
values['imdb'] = ''
values['tvdb'] = ''
values['metacache'] = False
self.list.append(values)
except:
log_utils.error()
def items_list(i):
if self.list[i]['metacache']: return
try:
values = {}
tmdb = self.list[i].get('tmdb', '')
# movie_meta = self.get_movie_meta(tmdb)
movie_meta = cache.get(self.get_movie_meta, 96, tmdb)
values.update(movie_meta)
imdb = values['imdb']
if not self.disable_fanarttv:
extended_art = cache.get(fanarttv.get_movie_art, 168, imdb, tmdb)
if extended_art: values.update(extended_art)
values = dict((k,v) for k, v in control.iteritems(values) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values.
self.list[i].update(values)
meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.user, 'item': values}
self.meta.append(meta)
except:
log_utils.error()
self.list = metacache.fetch(self.list, self.lang, self.user)
threads = []
for i in range(0, len(self.list)):
threads.append(workers.Thread(items_list, i))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache?
metacache.insert(self.meta)
sorted_list = []
self.list = [i for i in self.list if i.get('tmdb')]
for i in sortList:
sorted_list += [item for item in self.list if item['tmdb'] == i] # resort to match TMDb list because threading will loose order.
return sorted_list
def tmdb_collections_list(self, url):
if not url: return
try:
result = get_request(url)
if '/3/' in url: items = result['items']
else: items = result['results']
except: return
self.list = []
try:
page = int(result['page'])
total = int(result['total_pages'])
if page >= total: raise Exception()
if 'page=' not in url: raise Exception()
next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1)
except: next = ''
for item in items:
try:
values = {}
values['next'] = next
media_type = item.get('media_type', '')
if media_type == 'movie': continue
values['tmdb'] = str(item.get('id', '')) if item.get('id') else ''
values['metacache'] = False
self.list.append(values)
except:
log_utils.error()
def items_list(i):
if self.list[i]['metacache']: return
try:
values = {}
tmdb = self.list[i].get('tmdb', '')
# showSeasons_meta =self.get_showSeasons_meta(tmdb)
showSeasons_meta = cache.get(self.get_showSeasons_meta, 96, tmdb)
values.update(showSeasons_meta)
imdb = values['imdb']
tvdb = values['tvdb']
if not self.disable_fanarttv:
extended_art = cache.get(fanarttv.get_tvshow_art, 168, tvdb)
if extended_art: values.update(extended_art)
values = dict((k,v) for k, v in control.iteritems(values) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values.
self.list[i].update(values)
meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb, 'lang': self.lang, 'user': self.user, 'item': values}
self.meta.append(meta)
except:
log_utils.error()
self.list = metacache.fetch(self.list, self.lang, self.user)
threads = []
for i in range(0, len(self.list)):
threads.append(workers.Thread(items_list, i))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache?
metacache.insert(self.meta)
self.list = [i for i in self.list if i.get('tmdb')]
return self.list
def worker(self, level=1):
try:
if not self.list: return
self.meta = []
total = len(self.list)
for i in range(0, total):
self.list[i].update({'metacache': False})
self.list = metacache.fetch(self.list, self.lang, self.user)
for r in range(0, total, 40):
threads = []
for i in range(r, r + 40):
if i < total: threads.append(workers.Thread(self.super_info, i))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache?
metacache.insert(self.meta)
self.list = [i for i in self.list if i.get('tmdb')]
except:
log_utils.error()
def imdb_list(self, url, isRatinglink=False):
list = []
try:
for i in re.findall(r'date\[(\d+)\]', url):
url = url.replace('date[%s]' % i, (self.date_time - timedelta(days=int(i))).strftime('%Y-%m-%d'))
def imdb_watchlist_id(url):
return client.parseDOM(client.request(url), 'meta', ret='content', attrs = {'property': 'pageId'})[0]
# return client.parseDOM(client.request(url).decode('iso-8859-1').encode('utf-8'), 'meta', ret='content', attrs = {'property': 'pageId'})[0]
if url == self.imdbwatchlist_link:
url = cache.get(imdb_watchlist_id, 8640, url)
url = self.imdblist_link % url
result = client.request(url)
result = result.replace('\n', ' ')
# result = result.decode('iso-8859-1').encode('utf-8')
items = client.parseDOM(result, 'div', attrs = {'class': '.+? lister-item'}) + client.parseDOM(result, 'div', attrs = {'class': 'lister-item .+?'})
items += client.parseDOM(result, 'div', attrs = {'class': 'list_item.+?'})
except:
log_utils.error()
return
next = ''
try:
# HTML syntax error, " directly followed by attribute name. Insert space in between. parseDOM can otherwise not handle it.
result = result.replace('"class="lister-page-next', '" class="lister-page-next')
# next = client.parseDOM(result, 'a', ret='href', attrs = {'class': '.+?ister-page-nex.+?'})
next = client.parseDOM(result, 'a', ret='href', attrs = {'class': 'lister-page-next.+?'})
if len(next) == 0:
next = client.parseDOM(result, 'div', attrs = {'class': 'pagination'})[0]
next = zip(client.parseDOM(next, 'a', ret='href'), client.parseDOM(next, 'a'))
next = [i[0] for i in next if 'Next' in i[1]]
next = url.replace(urlparse(url).query, urlparse(next[0]).query)
next = client.replaceHTMLCodes(next)
except:
next = ''
for item in items:
try:
title = client.replaceHTMLCodes(client.parseDOM(item, 'a')[1])
year = client.parseDOM(item, 'span', attrs = {'class': 'lister-item-year.+?'})
try: year = re.findall(r'(\d{4})', year[0])[0]
except: continue
if int(year) > int((self.date_time).strftime('%Y')): continue
try: show = '–'.decode('utf-8') in str(year).decode('utf-8') or '-'.decode('utf-8') in str(year).decode('utf-8') # check with Matrix
except: show = False
if show or 'Episode:' in item: raise Exception() # Some lists contain TV shows.
try: genre = client.parseDOM(item, 'span', attrs = {'class': 'genre'})[0]
except: genre = ''
genre = ' / '.join([i.strip() for i in genre.split(',')])
genre = client.replaceHTMLCodes(genre)
try: mpaa = client.parseDOM(item, 'span', attrs = {'class': 'certificate'})[0]
except: mpaa = ''
if isRatinglink and 'Short' not in genre:
if mpaa in ['TV-Y', 'TV-Y7', 'TV-G', 'TV-PG', 'TV-13', 'TV-14', 'TV-MA']:
raise Exception()
if mpaa == '' or mpaa == 'NOT_RATED': mpaa = ''
mpaa = mpaa.replace('_', '-')
mpaa = client.replaceHTMLCodes(mpaa)
imdb = client.parseDOM(item, 'a', ret='href')[0]
imdb = re.findall(r'(tt\d*)', imdb)[0]
try: # parseDOM cannot handle elements without a closing tag.
from bs4 import BeautifulSoup
html = BeautifulSoup(item, "html.parser")
poster = html.find_all('img')[0]['loadlate']
except: poster = ''
if '/nopicture/' in poster: poster = ''
if poster:
poster = re.sub(r'(?:_SX|_SY|_UX|_UY|_CR|_AL)(?:\d+|_).+?\.', '_SX500.', poster)
poster = client.replaceHTMLCodes(poster)
try: duration = re.findall(r'(\d+?) min(?:s|)', item)[-1]
except: duration = ''
rating = ''
try: rating = client.parseDOM(item, 'span', attrs = {'class': 'rating-rating'})[0]
except:
try: rating = client.parseDOM(rating, 'span', attrs = {'class': 'value'})[0]
except:
try: rating = client.parseDOM(item, 'div', ret='data-value', attrs = {'class': '.*?imdb-rating'})[0]
except: pass
if rating == '-': rating = ''
if not rating:
try:
rating = client.parseDOM(item, 'span', attrs = {'class': 'ipl-rating-star__rating'})[0]
if rating == '-': rating = ''
except: pass
rating = client.replaceHTMLCodes(rating)
votes = ''
try: votes = client.parseDOM(item, 'span', attrs = {'name': 'nv'})[0]
except:
try: votes = client.parseDOM(item, 'div', ret='title', attrs = {'class': '.*?rating-list'})[0]
except:
try: votes = re.findall(r'\((.+?) vote(?:s|)\)', votes)[0]
except: pass
votes = client.replaceHTMLCodes(votes)
try: director = re.findall(r'Director(?:s|):(.+?)(?:\||</div>)', item)[0]
except: director = ''
director = client.parseDOM(director, 'a')
director = ' / '.join(director)
director = client.replaceHTMLCodes(director)
plot = ''
try: plot = client.parseDOM(item, 'p', attrs = {'class': 'text-muted'})[0]
except:
try: plot = client.parseDOM(item, 'div', attrs = {'class': 'item_description'})[0]
except: pass
plot = plot.rsplit('<span>', 1)[0].strip()
plot = re.sub(r'<.+?>|</.+?>', '', plot)
if not plot:
try:
plot = client.parseDOM(item, 'div', attrs = {'class': 'lister-item-content'})[0] # not sure on this, check html
plot = re.sub(r'<p\s*class="">', '<p class="plot_">', plot)
plot = client.parseDOM(plot, 'p', attrs = {'class': 'plot_'})[0]
plot = re.sub(r'<.+?>|</.+?>', '', plot)
except: pass
plot = client.cleanHTML(plot)
values = {}
values = {'content': 'movie', 'title': title, 'originaltitle': title, 'year': year, 'premiered': premiered,
'studio': '', 'genre': genre, 'duration': duration, 'rating': rating, 'votes': votes, 'mpaa': mpaa,
'director': director, 'writer': writer, 'cast': cast, 'plot': plot, 'code': tmdb,
'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'poster': poster, 'poster2': '', 'poster3': '', 'banner': '',
'fanart': '', 'fanart2': '', 'fanart3': '', 'clearlogo': '', 'clearart': '', 'landscape': '',
'metacache': False, 'next': next}
meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.tmdb_key, 'item': values}
if not self.disable_fanarttv:
extended_art = cache.get(fanarttv.get_movie_art, 168, imdb, tmdb)
if extended_art:
values.update(extended_art)
meta.update(values)
values = dict((k, v) for k, v in iter(values.items()) if v is not None and v != '')
self.list.append(values)
if 'next' in meta.get('item'): del meta['item']['next'] # next can not exist in metacache
self.meta.append(meta)
self.meta = [i for i in self.meta if i.get('tmdb')] # without this ui removed missing tmdb but it still writes these cases to metacache?
metacache.insert(self.meta) # don't insert until cleaned up and can also be fetched.
except:
log_utils.error()
return self.list
def items_list(tvmaze_id):
# if i['metacache']: return # not possible with only a tvmaze_id
try:
values = {}
values['next'] = next
values['tvmaze'] = tvmaze_id
url = self.tvmaze_info_link % tvmaze_id
item = get_request(url)
values['content'] = item.get('type', '').lower()
values['mediatype'] = 'tvshow'
values['title'] = item.get('name')
values['originaltitle'] = values['title']
values['tvshowtitle'] = values['title']
values['premiered'] = str(item.get(
'premiered', '')) if item.get('premiered') else ''
try:
values['year'] = values['premiered'][:4]
except:
values['year'] = ''
ids = item.get('externals')
imdb = str(ids.get('imdb', '')) if ids.get('imdb') else ''
tvdb = str(ids.get('thetvdb',
'')) if ids.get('thetvdb') else ''
tmdb = '' # TVMaze does not have tmdb_id in api
studio = item.get('network', {}) or item.get('webChannel', {})
values['studio'] = studio.get('name', '')
values['genre'] = []
for i in item['genres']:
values['genre'].append(i.title())
if values['genre'] == []: values['genre'] = 'NA'
values['duration'] = int(item.get(
'runtime', '')) * 60 if item.get('runtime') else ''
values['rating'] = str(item.get('rating').get(
'average',
'')) if item.get('rating').get('average') else ''
values['plot'] = client.cleanHTML(item['summary'])
values['status'] = item.get('status', '')
values['castandart'] = []
for person in item['_embedded']['cast']:
try:
values['castandart'].append({
'name':
person['person']['name'],
'role':
person['character']['name'],
'thumbnail':
(person['person']['image']['medium']
if person['person']['image']['medium'] else '')
})
except:
pass
if len(values['castandart']) == 150: break
image = item.get('image', {}) or ''
values['poster'] = image.get('original', '') if image else ''
values['fanart'] = ''
values['banner'] = ''
values['mpaa'] = ''
values['votes'] = ''
try:
values['airday'] = item['schedule']['days'][0]
except:
values['airday'] = ''
values['airtime'] = item['schedule']['time'] or ''
try:
values['airzone'] = item['network']['country']['timezone']
except:
values['airzone'] = ''
values['metacache'] = False
#### -- Missing id's lookup -- ####
if not tmdb and (imdb or tvdb):
try:
result = cache.get(tmdb_indexer.TVshows().IdLookup, 96,
imdb, tvdb)
tmdb = str(result.get('id',
'')) if result.get('id') else ''
except:
tmdb = ''
if not imdb or not tmdb or not tvdb:
try:
trakt_ids = trakt.SearchTVShow(quote_plus(
values['tvshowtitle']),
values['year'],
full=False)
if not trakt_ids: raise Exception
ids = trakt_ids[0].get('show', {}).get('ids', {})
if not imdb:
imdb = str(ids.get('imdb',
'')) if ids.get('imdb') else ''
if not tmdb:
tmdb = str(ids.get('tmdb',
'')) if ids.get('tmdb') else ''
if not tvdb:
tvdb = str(ids.get('tvdb',
'')) if ids.get('tvdb') else ''
except:
log_utils.error()
#################################
if not tmdb:
return log_utils.log(
'tvshowtitle: (%s) missing tmdb_id: ids={imdb: %s, tmdb: %s, tvdb: %s}'
% (values['tvshowtitle'], imdb, tmdb, tvdb), __name__,
log_utils.LOGDEBUG
) # log TMDb shows that they do not have
# self.list = metacache.fetch(self.list, self.lang, self.user)
# if self.list['metacache'] is True: raise Exception()
showSeasons = cache.get(
tmdb_indexer.TVshows().get_showSeasons_meta, 96, tmdb)
if not showSeasons: return
showSeasons = dict(
(k, v) for k, v in iter(showSeasons.items())
if v is not None and v != ''
) # removes empty keys so .update() doesn't over-write good meta
values.update(showSeasons)
if not values.get('imdb'): values['imdb'] = imdb
if not values.get('tmdb'): values['tmdb'] = tmdb
if not values.get('tvdb'): values['tvdb'] = tvdb
for k in ('seasons', ):
values.pop(
k, None
) # pop() keys from showSeasons that are not needed anymore
if self.enable_fanarttv:
extended_art = fanarttv_cache.get(fanarttv.get_tvshow_art,
168, tvdb)
if extended_art: values.update(extended_art)
meta = {
'imdb': imdb,
'tmdb': tmdb,
'tvdb': tvdb,
'lang': self.lang,
'user': self.user,
'item': values
} # DO NOT move this after "values = dict()" below or it becomes the same object and "del meta['item']['next']" removes it from both
values = dict((k, v) for k, v in iter(values.items())
if v is not None and v != '')
self.list.append(values)
if 'next' in meta.get('item'):
del meta['item']['next'] # next can not exist in metacache
self.meta.append(meta)
self.meta = [
i for i in self.meta if i.get('tmdb')
] # without this ui removed missing tmdb but it still writes these cases to metacache?
metacache.insert(self.meta)
except:
log_utils.error()
def tmdb_collections_list(self, url):
if not url: return
try:
result = get_request(url)
if '404:NOT FOUND' in result: return result
if '/collection/' in url: items = result['parts']
elif '/3/' in url: items = result['items']
else: items = result['results']
except: return
self.list = []
try:
page = int(result['page'])
total = int(result['total_pages'])
if page >= total: raise Exception()
if 'page=' not in url: raise Exception()
next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1)
except: next = ''
for item in items:
try:
values = {}
values['next'] = next
media_type = item.get('media_type', '')
if media_type == 'movie': continue
values['tmdb'] = str(item.get('id', '')) if item.get('id') else ''
values['metacache'] = False
self.list.append(values)
except:
from resources.lib.modules import log_utils
log_utils.error()
def items_list(i):
if self.list[i]['metacache']: return
try:
values = {}
tmdb = self.list[i].get('tmdb', '')
showSeasons_meta = cache.get(self.get_showSeasons_meta, 96, tmdb)
values.update(showSeasons_meta)
imdb = values['imdb']
tvdb = values['tvdb']
if self.enable_fanarttv:
extended_art = fanarttv_cache.get(fanarttv.get_tvshow_art, 168, tvdb)
if extended_art: values.update(extended_art)
values = dict((k,v) for k, v in iter(values.items()) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values.
self.list[i].update(values)
meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb, 'lang': self.lang, 'user': self.user, 'item': values}
self.meta.append(meta)
except:
from resources.lib.modules import log_utils
log_utils.error()
self.list = metacache.fetch(self.list, self.lang, self.user)
threads = []
for i in range(0, len(self.list)):
threads.append(Thread(target=items_list, args=(i,)))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
self.meta = [i for i in self.meta if i.get('tmdb')]
metacache.insert(self.meta)
self.list = [i for i in self.list if i.get('tmdb')]
return self.list
def tmdb_list(self, url):
try:
result = get_request(url % API_key)
if '404:NOT FOUND' in result: return result
items = result['results']
except: return
self.list = [] ; sortList = []
try:
page = int(result['page'])
total = int(result['total_pages'])
if page >= total: raise Exception()
if 'page=' not in url: raise Exception()
next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1)
except: next = ''
for item in items:
try:
values = {}
values['next'] = next
values['tmdb'] = str(item.get('id', '')) if item.get('id') else ''
sortList.append(values['tmdb'])
values['imdb'] = ''
values['tvdb'] = ''
values['metacache'] = False
self.list.append(values)
except:
from resources.lib.modules import log_utils
log_utils.error()
def items_list(i):
if self.list[i]['metacache']: return
try:
values = {}
tmdb = self.list[i].get('tmdb', '')
movie_meta = cache.get(self.get_movie_meta, 96, tmdb)
values.update(movie_meta)
imdb = values['imdb']
if self.enable_fanarttv:
extended_art = fanarttv_cache.get(fanarttv.get_movie_art, 168, imdb, tmdb)
if extended_art: values.update(extended_art)
values = dict((k,v) for k, v in iter(values.items()) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values.
self.list[i].update(values)
meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.user, 'item': values}
self.meta.append(meta)
except:
from resources.lib.modules import log_utils
log_utils.error()
self.list = metacache.fetch(self.list, self.lang, self.user)
threads = []
for i in range(0, len(self.list)):
threads.append(Thread(target=items_list, args=(i,)))
[i.start() for i in threads]
[i.join() for i in threads]
if self.meta:
self.meta = [i for i in self.meta if i.get('tmdb')]
metacache.insert(self.meta)
sorted_list = []
self.list = [i for i in self.list if i.get('tmdb')]
for i in sortList:
sorted_list += [item for item in self.list if item['tmdb'] == i] # resort to match TMDb list because threading will lose order.
return sorted_list
