def worker(self): if not control.setting('metadata') == 'true': return self.imdb_info_link = 'https://www.omdbapi.com/?i=%s&plot=full&r=json' self.tvmaze_info_link = 'https://api.tvmaze.com/lookup/shows?thetvdb=%s' self.lang = 'en' self.meta = [] total = len(self.list) if total == 0: return for i in range(0, total): self.list[i].update({'metacache': False}) self.list = metacache.fetch(self.list, self.lang) multi = [i['imdb'] for i in self.list] multi = [x for y,x in enumerate(multi) if x not in multi[:y]] if len(multi) == 1: self.movie_info(0) ; self.tv_info(0) if self.meta: metacache.insert(self.meta) for i in range(0, total): self.list[i].update({'metacache': False}) self.list = metacache.fetch(self.list, self.lang) for r in range(0, total, 50): threads = [] for i in range(r, r+50): if i <= total: threads.append(workers.Thread(self.movie_info, i)) if i <= total: threads.append(workers.Thread(self.tv_info, i)) [i.start() for i in threads] [i.join() for i in threads] if self.meta: metacache.insert(self.meta)
def tmdb_list(self, url): try: result = get_request(url % API_key) items = result['results'] except: return self.list = [] ; sortList = [] try: page = int(result['page']) total = int(result['total_pages']) if page >= total: raise Exception() if 'page=' not in url: raise Exception() next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1) except: next = '' for item in items: try: values = {} values['next'] = next values['tmdb'] = str(item.get('id', '')) if item.get('id') else '' sortList.append(values['tmdb']) values['imdb'] = '' values['tvdb'] = '' values['metacache'] = False self.list.append(values) except: log_utils.error() def items_list(i): if self.list[i]['metacache']: return try: values = {} tmdb = self.list[i].get('tmdb', '') # movie_meta = self.get_movie_meta(tmdb) movie_meta = cache.get(self.get_movie_meta, 96, tmdb) values.update(movie_meta) imdb = values['imdb'] if not self.disable_fanarttv: extended_art = cache.get(fanarttv.get_movie_art, 168, imdb, tmdb) if extended_art: values.update(extended_art) values = dict((k,v) for k, v in control.iteritems(values) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values. self.list[i].update(values) meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.user, 'item': values} self.meta.append(meta) except: log_utils.error() self.list = metacache.fetch(self.list, self.lang, self.user) threads = [] for i in range(0, len(self.list)): threads.append(workers.Thread(items_list, i)) [i.start() for i in threads] [i.join() for i in threads] if self.meta: self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache? metacache.insert(self.meta) sorted_list = [] self.list = [i for i in self.list if i.get('tmdb')] for i in sortList: sorted_list += [item for item in self.list if item['tmdb'] == i] # resort to match TMDb list because threading will loose order. return sorted_list
def tmdb_collections_list(self, url): if not url: return try: result = get_request(url) if '/3/' in url: items = result['items'] else: items = result['results'] except: return self.list = [] try: page = int(result['page']) total = int(result['total_pages']) if page >= total: raise Exception() if 'page=' not in url: raise Exception() next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1) except: next = '' for item in items: try: values = {} values['next'] = next media_type = item.get('media_type', '') if media_type == 'movie': continue values['tmdb'] = str(item.get('id', '')) if item.get('id') else '' values['metacache'] = False self.list.append(values) except: log_utils.error() def items_list(i): if self.list[i]['metacache']: return try: values = {} tmdb = self.list[i].get('tmdb', '') # showSeasons_meta =self.get_showSeasons_meta(tmdb) showSeasons_meta = cache.get(self.get_showSeasons_meta, 96, tmdb) values.update(showSeasons_meta) imdb = values['imdb'] tvdb = values['tvdb'] if not self.disable_fanarttv: extended_art = cache.get(fanarttv.get_tvshow_art, 168, tvdb) if extended_art: values.update(extended_art) values = dict((k,v) for k, v in control.iteritems(values) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values. self.list[i].update(values) meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb, 'lang': self.lang, 'user': self.user, 'item': values} self.meta.append(meta) except: log_utils.error() self.list = metacache.fetch(self.list, self.lang, self.user) threads = [] for i in range(0, len(self.list)): threads.append(workers.Thread(items_list, i)) [i.start() for i in threads] [i.join() for i in threads] if self.meta: self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache? metacache.insert(self.meta) self.list = [i for i in self.list if i.get('tmdb')] return self.list
def worker(self, level=1): try: if not self.list: return self.meta = [] total = len(self.list) for i in range(0, total): self.list[i].update({'metacache': False}) self.list = metacache.fetch(self.list, self.lang, self.user) for r in range(0, total, 40): threads = [] for i in range(r, r + 40): if i < total: threads.append(workers.Thread(self.super_info, i)) [i.start() for i in threads] [i.join() for i in threads] if self.meta: self.meta = [i for i in self.meta if i.get('tmdb')] # without this "self.list=" below removes missing tmdb but here still writes these cases to metacache? metacache.insert(self.meta) self.list = [i for i in self.list if i.get('tmdb')] except: log_utils.error()
def imdb_list(self, url, isRatinglink=False): list = [] try: for i in re.findall(r'date\[(\d+)\]', url): url = url.replace('date[%s]' % i, (self.date_time - timedelta(days=int(i))).strftime('%Y-%m-%d')) def imdb_watchlist_id(url): return client.parseDOM(client.request(url), 'meta', ret='content', attrs = {'property': 'pageId'})[0] # return client.parseDOM(client.request(url).decode('iso-8859-1').encode('utf-8'), 'meta', ret='content', attrs = {'property': 'pageId'})[0] if url == self.imdbwatchlist_link: url = cache.get(imdb_watchlist_id, 8640, url) url = self.imdblist_link % url result = client.request(url) result = result.replace('\n', ' ') # result = result.decode('iso-8859-1').encode('utf-8') items = client.parseDOM(result, 'div', attrs = {'class': '.+? lister-item'}) + client.parseDOM(result, 'div', attrs = {'class': 'lister-item .+?'}) items += client.parseDOM(result, 'div', attrs = {'class': 'list_item.+?'}) except: log_utils.error() return next = '' try: # HTML syntax error, " directly followed by attribute name. Insert space in between. parseDOM can otherwise not handle it. result = result.replace('"class="lister-page-next', '" class="lister-page-next') # next = client.parseDOM(result, 'a', ret='href', attrs = {'class': '.+?ister-page-nex.+?'}) next = client.parseDOM(result, 'a', ret='href', attrs = {'class': 'lister-page-next.+?'}) if len(next) == 0: next = client.parseDOM(result, 'div', attrs = {'class': 'pagination'})[0] next = zip(client.parseDOM(next, 'a', ret='href'), client.parseDOM(next, 'a')) next = [i[0] for i in next if 'Next' in i[1]] next = url.replace(urlparse(url).query, urlparse(next[0]).query) next = client.replaceHTMLCodes(next) except: next = '' for item in items: try: title = client.replaceHTMLCodes(client.parseDOM(item, 'a')[1]) year = client.parseDOM(item, 'span', attrs = {'class': 'lister-item-year.+?'}) try: year = re.findall(r'(\d{4})', year[0])[0] except: continue if int(year) > int((self.date_time).strftime('%Y')): continue try: show = '–'.decode('utf-8') in str(year).decode('utf-8') or '-'.decode('utf-8') in str(year).decode('utf-8') # check with Matrix except: show = False if show or 'Episode:' in item: raise Exception() # Some lists contain TV shows. try: genre = client.parseDOM(item, 'span', attrs = {'class': 'genre'})[0] except: genre = '' genre = ' / '.join([i.strip() for i in genre.split(',')]) genre = client.replaceHTMLCodes(genre) try: mpaa = client.parseDOM(item, 'span', attrs = {'class': 'certificate'})[0] except: mpaa = '' if isRatinglink and 'Short' not in genre: if mpaa in ['TV-Y', 'TV-Y7', 'TV-G', 'TV-PG', 'TV-13', 'TV-14', 'TV-MA']: raise Exception() if mpaa == '' or mpaa == 'NOT_RATED': mpaa = '' mpaa = mpaa.replace('_', '-') mpaa = client.replaceHTMLCodes(mpaa) imdb = client.parseDOM(item, 'a', ret='href')[0] imdb = re.findall(r'(tt\d*)', imdb)[0] try: # parseDOM cannot handle elements without a closing tag. from bs4 import BeautifulSoup html = BeautifulSoup(item, "html.parser") poster = html.find_all('img')[0]['loadlate'] except: poster = '' if '/nopicture/' in poster: poster = '' if poster: poster = re.sub(r'(?:_SX|_SY|_UX|_UY|_CR|_AL)(?:\d+|_).+?\.', '_SX500.', poster) poster = client.replaceHTMLCodes(poster) try: duration = re.findall(r'(\d+?) min(?:s|)', item)[-1] except: duration = '' rating = '' try: rating = client.parseDOM(item, 'span', attrs = {'class': 'rating-rating'})[0] except: try: rating = client.parseDOM(rating, 'span', attrs = {'class': 'value'})[0] except: try: rating = client.parseDOM(item, 'div', ret='data-value', attrs = {'class': '.*?imdb-rating'})[0] except: pass if rating == '-': rating = '' if not rating: try: rating = client.parseDOM(item, 'span', attrs = {'class': 'ipl-rating-star__rating'})[0] if rating == '-': rating = '' except: pass rating = client.replaceHTMLCodes(rating) votes = '' try: votes = client.parseDOM(item, 'span', attrs = {'name': 'nv'})[0] except: try: votes = client.parseDOM(item, 'div', ret='title', attrs = {'class': '.*?rating-list'})[0] except: try: votes = re.findall(r'\((.+?) vote(?:s|)\)', votes)[0] except: pass votes = client.replaceHTMLCodes(votes) try: director = re.findall(r'Director(?:s|):(.+?)(?:\||</div>)', item)[0] except: director = '' director = client.parseDOM(director, 'a') director = ' / '.join(director) director = client.replaceHTMLCodes(director) plot = '' try: plot = client.parseDOM(item, 'p', attrs = {'class': 'text-muted'})[0] except: try: plot = client.parseDOM(item, 'div', attrs = {'class': 'item_description'})[0] except: pass plot = plot.rsplit('<span>', 1)[0].strip() plot = re.sub(r'<.+?>|</.+?>', '', plot) if not plot: try: plot = client.parseDOM(item, 'div', attrs = {'class': 'lister-item-content'})[0] # not sure on this, check html plot = re.sub(r'<p\s*class="">', '<p class="plot_">', plot) plot = client.parseDOM(plot, 'p', attrs = {'class': 'plot_'})[0] plot = re.sub(r'<.+?>|</.+?>', '', plot) except: pass plot = client.cleanHTML(plot) values = {} values = {'content': 'movie', 'title': title, 'originaltitle': title, 'year': year, 'premiered': premiered, 'studio': '', 'genre': genre, 'duration': duration, 'rating': rating, 'votes': votes, 'mpaa': mpaa, 'director': director, 'writer': writer, 'cast': cast, 'plot': plot, 'code': tmdb, 'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'poster': poster, 'poster2': '', 'poster3': '', 'banner': '', 'fanart': '', 'fanart2': '', 'fanart3': '', 'clearlogo': '', 'clearart': '', 'landscape': '', 'metacache': False, 'next': next} meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.tmdb_key, 'item': values} if not self.disable_fanarttv: extended_art = cache.get(fanarttv.get_movie_art, 168, imdb, tmdb) if extended_art: values.update(extended_art) meta.update(values) values = dict((k, v) for k, v in iter(values.items()) if v is not None and v != '') self.list.append(values) if 'next' in meta.get('item'): del meta['item']['next'] # next can not exist in metacache self.meta.append(meta) self.meta = [i for i in self.meta if i.get('tmdb')] # without this ui removed missing tmdb but it still writes these cases to metacache? metacache.insert(self.meta) # don't insert until cleaned up and can also be fetched. except: log_utils.error() return self.list
def items_list(tvmaze_id): # if i['metacache']: return # not possible with only a tvmaze_id try: values = {} values['next'] = next values['tvmaze'] = tvmaze_id url = self.tvmaze_info_link % tvmaze_id item = get_request(url) values['content'] = item.get('type', '').lower() values['mediatype'] = 'tvshow' values['title'] = item.get('name') values['originaltitle'] = values['title'] values['tvshowtitle'] = values['title'] values['premiered'] = str(item.get( 'premiered', '')) if item.get('premiered') else '' try: values['year'] = values['premiered'][:4] except: values['year'] = '' ids = item.get('externals') imdb = str(ids.get('imdb', '')) if ids.get('imdb') else '' tvdb = str(ids.get('thetvdb', '')) if ids.get('thetvdb') else '' tmdb = '' # TVMaze does not have tmdb_id in api studio = item.get('network', {}) or item.get('webChannel', {}) values['studio'] = studio.get('name', '') values['genre'] = [] for i in item['genres']: values['genre'].append(i.title()) if values['genre'] == []: values['genre'] = 'NA' values['duration'] = int(item.get( 'runtime', '')) * 60 if item.get('runtime') else '' values['rating'] = str(item.get('rating').get( 'average', '')) if item.get('rating').get('average') else '' values['plot'] = client.cleanHTML(item['summary']) values['status'] = item.get('status', '') values['castandart'] = [] for person in item['_embedded']['cast']: try: values['castandart'].append({ 'name': person['person']['name'], 'role': person['character']['name'], 'thumbnail': (person['person']['image']['medium'] if person['person']['image']['medium'] else '') }) except: pass if len(values['castandart']) == 150: break image = item.get('image', {}) or '' values['poster'] = image.get('original', '') if image else '' values['fanart'] = '' values['banner'] = '' values['mpaa'] = '' values['votes'] = '' try: values['airday'] = item['schedule']['days'][0] except: values['airday'] = '' values['airtime'] = item['schedule']['time'] or '' try: values['airzone'] = item['network']['country']['timezone'] except: values['airzone'] = '' values['metacache'] = False #### -- Missing id's lookup -- #### if not tmdb and (imdb or tvdb): try: result = cache.get(tmdb_indexer.TVshows().IdLookup, 96, imdb, tvdb) tmdb = str(result.get('id', '')) if result.get('id') else '' except: tmdb = '' if not imdb or not tmdb or not tvdb: try: trakt_ids = trakt.SearchTVShow(quote_plus( values['tvshowtitle']), values['year'], full=False) if not trakt_ids: raise Exception ids = trakt_ids[0].get('show', {}).get('ids', {}) if not imdb: imdb = str(ids.get('imdb', '')) if ids.get('imdb') else '' if not tmdb: tmdb = str(ids.get('tmdb', '')) if ids.get('tmdb') else '' if not tvdb: tvdb = str(ids.get('tvdb', '')) if ids.get('tvdb') else '' except: log_utils.error() ################################# if not tmdb: return log_utils.log( 'tvshowtitle: (%s) missing tmdb_id: ids={imdb: %s, tmdb: %s, tvdb: %s}' % (values['tvshowtitle'], imdb, tmdb, tvdb), __name__, log_utils.LOGDEBUG ) # log TMDb shows that they do not have # self.list = metacache.fetch(self.list, self.lang, self.user) # if self.list['metacache'] is True: raise Exception() showSeasons = cache.get( tmdb_indexer.TVshows().get_showSeasons_meta, 96, tmdb) if not showSeasons: return showSeasons = dict( (k, v) for k, v in iter(showSeasons.items()) if v is not None and v != '' ) # removes empty keys so .update() doesn't over-write good meta values.update(showSeasons) if not values.get('imdb'): values['imdb'] = imdb if not values.get('tmdb'): values['tmdb'] = tmdb if not values.get('tvdb'): values['tvdb'] = tvdb for k in ('seasons', ): values.pop( k, None ) # pop() keys from showSeasons that are not needed anymore if self.enable_fanarttv: extended_art = fanarttv_cache.get(fanarttv.get_tvshow_art, 168, tvdb) if extended_art: values.update(extended_art) meta = { 'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb, 'lang': self.lang, 'user': self.user, 'item': values } # DO NOT move this after "values = dict()" below or it becomes the same object and "del meta['item']['next']" removes it from both values = dict((k, v) for k, v in iter(values.items()) if v is not None and v != '') self.list.append(values) if 'next' in meta.get('item'): del meta['item']['next'] # next can not exist in metacache self.meta.append(meta) self.meta = [ i for i in self.meta if i.get('tmdb') ] # without this ui removed missing tmdb but it still writes these cases to metacache? metacache.insert(self.meta) except: log_utils.error()
def tmdb_collections_list(self, url): if not url: return try: result = get_request(url) if '404:NOT FOUND' in result: return result if '/collection/' in url: items = result['parts'] elif '/3/' in url: items = result['items'] else: items = result['results'] except: return self.list = [] try: page = int(result['page']) total = int(result['total_pages']) if page >= total: raise Exception() if 'page=' not in url: raise Exception() next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1) except: next = '' for item in items: try: values = {} values['next'] = next media_type = item.get('media_type', '') if media_type == 'movie': continue values['tmdb'] = str(item.get('id', '')) if item.get('id') else '' values['metacache'] = False self.list.append(values) except: from resources.lib.modules import log_utils log_utils.error() def items_list(i): if self.list[i]['metacache']: return try: values = {} tmdb = self.list[i].get('tmdb', '') showSeasons_meta = cache.get(self.get_showSeasons_meta, 96, tmdb) values.update(showSeasons_meta) imdb = values['imdb'] tvdb = values['tvdb'] if self.enable_fanarttv: extended_art = fanarttv_cache.get(fanarttv.get_tvshow_art, 168, tvdb) if extended_art: values.update(extended_art) values = dict((k,v) for k, v in iter(values.items()) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values. self.list[i].update(values) meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': tvdb, 'lang': self.lang, 'user': self.user, 'item': values} self.meta.append(meta) except: from resources.lib.modules import log_utils log_utils.error() self.list = metacache.fetch(self.list, self.lang, self.user) threads = [] for i in range(0, len(self.list)): threads.append(Thread(target=items_list, args=(i,))) [i.start() for i in threads] [i.join() for i in threads] if self.meta: self.meta = [i for i in self.meta if i.get('tmdb')] metacache.insert(self.meta) self.list = [i for i in self.list if i.get('tmdb')] return self.list
def tmdb_list(self, url): try: result = get_request(url % API_key) if '404:NOT FOUND' in result: return result items = result['results'] except: return self.list = [] ; sortList = [] try: page = int(result['page']) total = int(result['total_pages']) if page >= total: raise Exception() if 'page=' not in url: raise Exception() next = '%s&page=%s' % (url.split('&page=', 1)[0], page+1) except: next = '' for item in items: try: values = {} values['next'] = next values['tmdb'] = str(item.get('id', '')) if item.get('id') else '' sortList.append(values['tmdb']) values['imdb'] = '' values['tvdb'] = '' values['metacache'] = False self.list.append(values) except: from resources.lib.modules import log_utils log_utils.error() def items_list(i): if self.list[i]['metacache']: return try: values = {} tmdb = self.list[i].get('tmdb', '') movie_meta = cache.get(self.get_movie_meta, 96, tmdb) values.update(movie_meta) imdb = values['imdb'] if self.enable_fanarttv: extended_art = fanarttv_cache.get(fanarttv.get_movie_art, 168, imdb, tmdb) if extended_art: values.update(extended_art) values = dict((k,v) for k, v in iter(values.items()) if v is not None and v != '') # remove empty keys so .update() doesn't over-write good meta with empty values. self.list[i].update(values) meta = {'imdb': imdb, 'tmdb': tmdb, 'tvdb': '', 'lang': self.lang, 'user': self.user, 'item': values} self.meta.append(meta) except: from resources.lib.modules import log_utils log_utils.error() self.list = metacache.fetch(self.list, self.lang, self.user) threads = [] for i in range(0, len(self.list)): threads.append(Thread(target=items_list, args=(i,))) [i.start() for i in threads] [i.join() for i in threads] if self.meta: self.meta = [i for i in self.meta if i.get('tmdb')] metacache.insert(self.meta) sorted_list = [] self.list = [i for i in self.list if i.get('tmdb')] for i in sortList: sorted_list += [item for item in self.list if item['tmdb'] == i] # resort to match TMDb list because threading will lose order. return sorted_list