def _fuzzy_title(a, b): ''' Determines how much of a is in b a (str): String to match against b b (str): String to match a against Order of a and b matters. A is broken down and words are compared against B's words. ie: _fuzzy_title('This is string a', 'This is string b and has extra words.') Returns 75 since 75% of a is in b. Returns int ''' a = a.replace('&', 'and') b = b.replace('&', 'and') a_words = Url.normalize(a).split(' ') b_words = Url.normalize(b).split(' ') m = 0 a_len = len(a_words) for i in a_words: if i in b_words: b_words.remove(i) m += 1 return int((m / a_len) * 100)
def fuzzy_title(self, titles): ''' Score and remove results based on title match titles (list): titles to match against If titles is an empty list every result is treated as a perfect match Iterates through self.results and removes any entry that does not fuzzy match 'title' > 70. Adds fuzzy_score / 20 points to ['score'] Does not return ''' logging.info('Checking title match.') lst = [] if titles == []: for result in self.results: result['score'] += 20 lst.append(result) else: for result in self.results: if result['type'] == 'import' and result not in lst: result['score'] += 20 lst.append(result) continue test = Url.normalize(result['title']) matches = [fuzz.partial_ratio(Url.normalize(title), test) for title in titles] if any([match > 70 for match in matches]): result['score'] += int(max(matches) / 5) lst.append(result) else: logging.debug('{} best title match was {}%, removing search result.'.format(test, max(matches))) self.results = lst logging.info('Keeping {} results.'.format(len(self.results)))
def get_imdbid(self, tmdbid=None, title=None, year=''): ''' Gets imdbid from tmdbid or title and year tmdbid: str TMDB movie id # title: str movie title year str year of movie release MUST supply either tmdbid or title. Year is optional with title, but results are more reliable with it. Returns str imdbid or None on failure ''' if not tmdbid and not title: logging.warning( 'Neither tmdbid or title supplied. Unable to find imdbid.') return None if not tmdbid: title = Url.normalize(title) year = Url.normalize(year) url = 'https://api.themoviedb.org/3/search/movie?api_key={}&language=en-US&query={}&year={}&page=1&include_adult=false'.format( _k(b'tmdb'), title, year) while self.get_tokens() < 3: sleep(0.3) self.use_token() try: results = json.loads(Url.open(url).text) results = results['results'] if results: tmdbid = results[0]['id'] else: return None except (SystemExit, KeyboardInterrupt): raise except Exception as e: # noqa logging.error('Error attempting to get TMDBID from TMDB.', exc_info=True) return None url = 'https://api.themoviedb.org/3/movie/{}?api_key={}'.format( tmdbid, _k(b'tmdb')) while self.get_tokens() < 3: sleep(0.3) self.use_token() try: results = json.loads(Url.open(url).text) return results.get('imdb_id') except Exception as e: # noqa logging.error('Error attempting to get IMDBID from TMDB.', exc_info=True) return None
def get_imdbid(tmdbid=None, title=None, year=''): ''' Gets imdbid from tmdbid or title and year tmdbid (str): themoviedatabase id # title (str): movie title year (str/int): year of movie release MUST supply either tmdbid or title. Year is optional with title, but results are more reliable with it. Returns str imdbid ''' if not tmdbid and not title: logging.warning( 'Neither tmdbid or title supplied. Unable to find imdbid.') return '' if not tmdbid: title = Url.normalize(title) year = Url.normalize(year) url = 'https://api.themoviedb.org/3/search/movie?api_key={}&language=en-US&query={}&year={}&page=1&include_adult={}'.format( _k(b'tmdb'), title, year, 'true' if core.CONFIG['Search']['allowadult'] else 'false') TheMovieDatabase._use_token() try: results = json.loads(Url.open(url).text) results = results['results'] if results: tmdbid = results[0]['id'] else: return '' except (SystemExit, KeyboardInterrupt): raise except Exception as e: logging.error('Error attempting to get TMDBID from TMDB.', exc_info=True) return '' url = 'https://api.themoviedb.org/3/movie/{}?api_key={}'.format( tmdbid, _k(b'tmdb')) TheMovieDatabase._use_token() try: results = json.loads(Url.open(url).text) return results.get('imdb_id') except Exception as e: logging.error('Error attempting to get IMDBID from TMDB.', exc_info=True) return ''
def fuzzy_title(self, titles): ''' Score and remove results based on title match titles (list): titles to match against If titles is an empty list every result is treated as a perfect match Iterates through self.results and removes any entry that does not fuzzy match 'title' > 70. Adds fuzzy_score / 20 points to ['score'] Does not return ''' logging.info('Checking title match.') lst = [] if titles == []: logging.debug( 'No titles available to compare, scoring all as perfect match.' ) for result in self.results: result['score'] += 20 lst.append(result) else: for result in self.results: if result['type'] == 'import' and result not in lst: logging.debug( '{} is an Import, soring as a perfect match.'.format( result['title'])) result['score'] += 20 lst.append(result) continue release = Url.normalize(result['title']) logging.debug('Comparing release {} with titles {}.'.format( result['title'], titles)) matches = [ lm.score(release, Url.normalize(title)) * 100 for title in titles ] if any(match > 70 for match in matches): result['score'] += int(max(matches) / 5) lst.append(result) else: logging.debug( '{} best title match was {}%, removing search result.'. format(release, max(matches))) self.results = lst logging.info('Keeping {} results.'.format(len(self.results)))
def trailer(title_date): ''' Gets trailer embed ID from Youtube. title_date (str): movie title and date ('Movie Title 2016') Attempts to connect 3 times in case Youtube is down or not responding Can fail if no response is received. Returns str ''' logging.info('Getting trailer url from YouTube for {}'.format(title_date)) search_term = Url.normalize((title_date + '+trailer')) url = 'https://www.googleapis.com/youtube/v3/search?part=snippet&q={}&maxResults=1&key={}'.format(search_term, _k(b'youtube')) tries = 0 while tries < 3: try: results = json.loads(Url.open(url).text) return results['items'][0]['id']['videoId'] except (SystemExit, KeyboardInterrupt): raise except Exception as e: if tries == 2: logging.error('Unable to get trailer from Youtube.', exc_info=True) tries += 1 return ''
def get_trailer(self, title_date): ''' Gets trailer embed url from Youtube. :param title_date: str movie title and date ("Movie Title 2016") Attempts to connect 3 times in case Youtube is down or not responding Can fail if no response is recieved. Returns str or None ''' search_term = Url.normalize((title_date + '+trailer')) url = u"https://www.googleapis.com/youtube/v3/search?part=snippet&q={}&maxResults=1&key={}".format( search_term, _k(b'youtube')) tries = 0 while tries < 3: try: results = json.loads(Url.open(url).text) return results['items'][0]['id']['videoId'] except (SystemExit, KeyboardInterrupt): raise except Exception as e: # noqa if tries == 2: logging.error('Unable to get trailer from Youtube.', exc_info=True) tries += 1 return None
def _search_title(self, title): ''' Search TMDB for title title (str): movie title Title can include year ie Move Title 2017 Returns list of results ''' title = Url.normalize(title) url = 'https://api.themoviedb.org/3/search/movie?page=1&include_adult=false&' if title[-4:].isdigit(): query = 'query={}&year={}'.format(title[:-5], title[-4:]) else: query = 'query={}'.format(title) url = url + query logging.info('Searching TMDB {}'.format(url)) url = url + '&api_key={}'.format(_k(b'tmdb')) self.use_token() try: results = json.loads(Url.open(url).text) if results.get('success') == 'false': return [] else: return results['results'][:6] except (SystemExit, KeyboardInterrupt): raise except Exception as e: logging.error('Error searching for title on TMDB.', exc_info=True) return []
def _search_db(title_year): ''' Helper for backlog_search title_year (str): movie title and year 'Black Swan 2010' Returns list of found predb entries ''' title_year = Url.normalize(title_year, ascii_only=True) categories = 'movies' if core.CONFIG['Search'].get('predb_unknown'): categories += ',unknown' url = 'http://predb.me/?cats={}&search={}&rss=1'.format( categories, title_year) try: response = Url.open(url).text results_xml = response.replace('&', '%26') items = _parse_predb_xml(results_xml) return items except (SystemExit, KeyboardInterrupt): raise except Exception as e: logging.error('Predb.me search failed.', exc_info=True) return []
def add_nzb(data): ''' Adds nzb file to sab to download :param data: dict of nzb information Returns dict {'response': True, 'downloadid': 'id'} {'response': False, 'error': 'exception'} ''' conf = core.CONFIG['Downloader']['Usenet']['Sabnzbd'] host = conf['host'] port = conf['port'] api = conf['api'] base_url = 'http://{}:{}/sabnzbd/api?apikey={}'.format(host, port, api) mode = 'addurl' name = urllib.parse.quote(data['guid']) nzbname = Url.normalize(data['title']) cat = conf['category'] priority_keys = { 'Paused': '-2', 'Low': '-1', 'Normal': '0', 'High': '1', 'Forced': '2' } priority = priority_keys[conf['priority']] command_url = '&mode={}&name={}&nzbname={}&cat={}&priority={}&output=json'.format( mode, name, nzbname, cat, priority) url = base_url + command_url try: response = json.loads(Url.open(url).text) if response['status'] is True and len(response['nzo_ids']) > 0: downloadid = response['nzo_ids'][0] logging.info( 'NZB sent to SABNzbd - downloadid {}.'.format(downloadid)) return {'response': True, 'downloadid': downloadid} else: logging.error( 'Unable to send NZB to Sabnzbd. {}'.format(response)) return {'response': False, 'error': 'Unable to add NZB.'} except Exception as e: logging.error('Unable to send NZB to Sabnzbd.', exc_info=True) return {'response': False, 'error': str(e)}
def _search_db(self, title_year): ''' Helper for backlog_search title_year (str): movie title and year 'Black Swan 2010' Returns list of found predb entries ''' title_year = Url.normalize(title_year) url = 'http://predb.me/?cats=movies&search={}&rss=1'.format(title_year) try: response = Url.open(url).text results_xml = response.replace('&', '%26') items = self._parse_predb_xml(results_xml) return items except (SystemExit, KeyboardInterrupt): raise except Exception as e: logging.error('Predb.me search failed.', exc_info=True) return []
def _fuzzy_match(self, predb_titles, title, year): ''' Fuzzy matches title with predb titles predb_titles (list): titles in predb response title (str): title to match to rss titles year (str): year of movie release Checks for any fuzzy match over 60% Returns bool ''' movie = Url.normalize('{}.{}'.format(title, year), ascii_only=True).replace(' ', '.') for pdb in predb_titles: if year not in pdb: continue pdb = pdb.split(year)[0] + year match = lm.score(pdb.replace(' ', '.'), movie) * 100 if match > 60: logging.debug('{} matches {} at {}%'.format(pdb, movie, int(match))) return True return False
def search_rss(self, title_year): ''' Searches predb rss for title_year :param title_year: str movie title and year 'Black Swan 2010' Returns list of found rss entries or None if not found. ''' title_year = Url.normalize(title_year) url = 'http://predb.me/?cats=movies&search={}&rss=1'.format(title_year) try: response = Url.open(url).text results_xml = response.replace('&', '%26') items = self.parse_predb_xml(results_xml) return items except (SystemExit, KeyboardInterrupt): raise except Exception as e: # noqa logging.error('Predb.me search failed.', exc_info=True) return None
def search_all(self, imdbid, title, year): ''' Performs backlog search for all indexers. imdbid (str): imdb id # title (str): movie title year (str/int): year of movie release Returns list of dicts with sorted release information. ''' torz_indexers = core.CONFIG['Indexers']['TorzNab'].values() self.imdbid = imdbid results = [] term = Url.normalize('{} {}'.format(title, year)) for indexer in torz_indexers: if indexer[2] is False: continue url_base = indexer[0] logging.info('Searching TorzNab indexer {}'.format(url_base)) if url_base[-1] != '/': url_base = url_base + '/' apikey = indexer[1] caps = core.sql.torznab_caps(url_base) if not caps: caps = self._get_caps(url_base, apikey) if caps is None: logging.error('Unable to get caps for {}'.format(url_base)) continue if 'imdbid' in caps: logging.info('{} supports imdbid search.'.format(url_base)) r = self.search_newznab(url_base, apikey, t='movie', cat=2000, imdbid=imdbid) else: logging.info( '{} does not support imdbid search, using q={}'.format( url_base, term)) r = self.search_newznab(url_base, apikey, t='search', cat=2000, q=term) for i in r: results.append(i) torrent_indexers = core.CONFIG['Indexers']['Torrent'] title = Url.normalize(title) year = Url.normalize(str(year)) if torrent_indexers['rarbg']: rarbg_results = Rarbg.search(imdbid) for i in rarbg_results: if i not in results: results.append(i) if torrent_indexers['limetorrents']: lime_results = LimeTorrents.search(imdbid, term) for i in lime_results: if i not in results: results.append(i) if torrent_indexers['skytorrents']: sky_results = SkyTorrents.search(imdbid, term) for i in sky_results: if i not in results: results.append(i) if torrent_indexers['torrentz2']: torrentz_results = Torrentz2.search(imdbid, term) for i in torrentz_results: if i not in results: results.append(i) if torrent_indexers['thepiratebay']: tpb_results = ThePirateBay.search(imdbid) for i in tpb_results: if i not in results: results.append(i) if torrent_indexers['yts']: yts_results = YTS.search(imdbid, term) for i in yts_results: if i not in results: results.append(i) if torrent_indexers['zooqle']: zooqle_results = Zooqle.search(imdbid, term) for i in zooqle_results: if i not in results: results.append(i) self.imdbid = None return results
def search_all(self, imdbid, title, year): ''' Performs backlog search for all indexers. imdbid (str): imdb id # title (str): movie title year (str/int): year of movie release Returns list of dicts with sorted release information. ''' torz_indexers = core.CONFIG['Indexers']['TorzNab'].values() results = [] term = Url.normalize('{} {}'.format(title, year)) for indexer in torz_indexers: if indexer[2] is False: continue url_base = indexer[0] logging.info('Searching TorzNab indexer {}'.format(url_base)) if url_base[-1] != '/': url_base = url_base + '/' apikey = indexer[1] caps = core.sql.torznab_caps(url_base) if not caps: caps = self._get_caps(url_base, apikey) if caps is None: logging.error('Unable to get caps for {}'.format(url_base)) continue if 'imdbid' in caps: logging.info('{} supports imdbid search.'.format(url_base)) r = self.search_newznab(url_base, apikey, 'movie', imdbid=imdbid) else: logging.info( '{} does not support imdbid search, using q={}'.format( url_base, term)) r = self.search_newznab(url_base, apikey, 'search', q=term, imdbid=imdbid) for i in r: results.append(i) for indexer, enabled in core.CONFIG['Indexers']['Torrent'].items(): if enabled: if not hasattr(torrent_modules, indexer): logging.warning( 'Torrent indexer {} enabled but not found in torrent_modules.' .format(indexer)) continue else: for i in getattr(torrent_modules, indexer).search(imdbid, term): if i not in results: results.append(i) return results
def search_all(self, imdbid, title, year): ''' Performs backlog search for all indexers. imdbid (str): imdb id # title (str): movie title year (str/int): year of movie release Returns list of dicts with sorted release information. ''' torz_indexers = core.CONFIG['Indexers']['TorzNab'].values() self.imdbid = imdbid results = [] term = Url.normalize('{} {}'.format(title, year)) for indexer in torz_indexers: if indexer[2] is False: continue url_base = indexer[0] logging.info('Searching TorzNab indexer {}'.format(url_base)) if url_base[-1] != '/': url_base = url_base + '/' apikey = indexer[1] caps = core.sql.torznab_caps(url_base) if not caps: caps = self._get_caps(url_base, apikey) if caps is None: logging.error('Unable to get caps for {}'.format(url_base)) continue if 'imdbid' in caps: logging.info('{} supports imdbid search.'.format(url_base)) r = self.search_newznab(url_base, apikey, t='movie', cat=2000, imdbid=imdbid) else: logging.info('{} does not support imdbid search, using q={}'.format(url_base, term)) r = self.search_newznab(url_base, apikey, t='search', cat=2000, q=term) for i in r: results.append(i) torrent_indexers = core.CONFIG['Indexers']['Torrent'] title = Url.normalize(title) year = Url.normalize(str(year)) logging.info("Starting search") for provider in TorrentProvider.__subclasses__(): if torrent_indexers[provider.id]: logging.info("Searching {}".format(provider.name)) indexer_result = provider.search(imdbid, term) for i in indexer_result: if i not in results: results.append(i) self.imdbid = None return results