def use_season(self, provider, info): """ Setup method to define season search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_opennic_dns", bool) and "opennic_dns_alias" in definition: definition = get_alias(definition, definition["opennic_dns_alias"]) season_query = definition['season_query'] if definition['season_query'] else '' log.debug("Season URL: %s%s" % (definition['base_url'], season_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_seasons')) self.max_size = get_float(get_setting('max_size_seasons')) self.check_sizes() self.info = info self.url = u"%s%s" % (definition['base_url'], season_query) if definition['season_keywords']: self.queries = ["%s" % definition['season_keywords']] self.extras = ["%s" % definition['season_extra'] if definition['season_extra'] else ''] if definition['season_keywords2']: self.queries.append("%s" % definition['season_keywords2']) self.extras.append("%s" % definition['season_extra2'] if definition['season_extra2'] else '')
def got_results(provider, results): """ Results callback once a provider found all its results, or not Args: provider (str): The provider ID results (list): The list of results """ global provider_names global provider_results global available_providers definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) max_results = get_setting('max_results', int) sorted_results = sorted(results, key=lambda r: (r['seeds']), reverse=True) if len(sorted_results) > max_results: sorted_results = sorted_results[:max_results] log.info(">> %s returned %2d results in %.1f seconds%s" % (definition['name'].rjust(longest), len(results), round(time.time() - request_time, 2), (", sending %d best ones" % max_results) if len(results) > max_results else "")) provider_results.extend(sorted_results) available_providers -= 1 if definition['name'] in provider_names: provider_names.remove(definition['name'])
def use_general(self, provider, payload): """ Setup method to define general search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) general_query = definition['general_query'] if definition[ 'general_query'] else '' log.debug("General URL: %s%s" % (definition['base_url'], general_query)) self.info = payload self.url = u"%s%s" % (definition['base_url'], general_query) if definition['general_keywords']: self.queries = [definition['general_keywords']] self.extras = [definition['general_extra']] if 'general_keywords_fallback' in definition and definition[ 'general_keywords_fallback']: self.queries.append(definition['general_keywords_fallback']) self.extras.append('-')
def __init__(self, info=None): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.user_agent = USER_AGENT self.info = info self.proxy_type = None self.proxy_url = None self.content = None self.status = None self.headers = dict() if get_setting("use_elementum_proxy", bool): elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum') if elementum_addon and elementum_addon.getSetting('internal_proxy_enabled') == "true": self.proxy_url = "{0}://{1}:{2}".format("http", "127.0.0.1", "65222") if info and "internal_proxy_url" in info: log.debug("Use Internal Elementum Proxy") self.proxy_url = info["internal_proxy_url"] if elementum_addon.getSetting("proxy_enabled") == "true" and get_setting("use_proxy_setting", bool): self.proxy_type = int(elementum_addon.getSetting("proxy_type")) log.debug("Use users proxy from elementum settings: {0}".format(proxy_types[self.proxy_type])) prx_host = elementum_addon.getSetting("proxy_host") prx_port = elementum_addon.getSetting("proxy_port") self.proxy_url = "{0}://{1}:{2}".format(proxy_types[self.proxy_type], prx_host, prx_port)
def use_season(self, provider, info): """ Setup method to define season search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) season_query = definition[ 'season_query'] if 'season_query' in definition and definition[ 'season_query'] else '' log.debug("[%s] Season URL: %s%s" % (provider, definition['base_url'], season_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_seasons')) self.max_size = get_float(get_setting('max_size_seasons')) self.check_sizes() self.info = info self.url = u"%s%s" % (definition['base_url'], season_query) self.collect_queries('season', definition)
def get_enabled_providers(method): """ Utility method to get all enabled provider IDs Returns: list: All available enabled provider IDs """ results = [] type = "2" if method == "general": type = "0" elif method == "movie": type = "1" for provider in definitions: if 'enabled' in definitions[ provider] and not definitions[provider]['enabled']: continue if get_setting('use_%s' % provider, bool): contains = get_setting('%s_contains' % provider, choices=('All', 'Movies', 'Shows')) if not contains or contains == "0": results.append(provider) elif contains == type: results.append(provider) if 'custom' in definitions[provider] and definitions[provider][ 'custom']: results.append(provider) return results
def use_anime(self, provider, info): """ Setup method to define anime search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) anime_query = definition[ 'anime_query'] if 'anime_query' in definition and definition[ 'anime_query'] else '' log.debug("[%s] Anime URL: %s%s" % (provider, definition['base_url'], anime_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_episodes')) self.max_size = get_float(get_setting('max_size_episodes')) self.check_sizes() self.info = info self.url = u"%s%s" % (definition['base_url'], anime_query) if self.info['absolute_number']: self.info['episode'] = self.info['absolute_number'] self.collect_queries('anime', definition)
def use_movie(self, provider, payload): """ Setup method to define movie search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) movie_query = definition['movie_query'] if definition[ 'movie_query'] else '' log.debug("Movies URL: %s%s" % (definition['base_url'], movie_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_movies')) self.max_size = get_float(get_setting('max_size_movies')) self.check_sizes() self.info = payload self.url = u"%s%s" % (definition['base_url'], movie_query) if definition['movie_keywords']: self.queries = ["%s" % definition['movie_keywords']] self.extras = ["%s" % definition['movie_extra']] if 'movie_keywords2' in definition and definition['movie_keywords2']: self.queries.append("%s" % definition['movie_keywords2']) self.extras.append("%s" % definition['movie_extra2'] if definition['movie_extra2'] else '') if 'movie_keywords_fallback' in definition and definition[ 'movie_keywords_fallback']: self.queries.append(definition['movie_keywords_fallback']) self.extras.append('-')
def use_episode(self, provider, payload): """ Setup method to define episode search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) show_query = definition['show_query'] if definition['show_query'] else '' log.debug("Episode URL: %s%s" % (definition['base_url'], show_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_episodes')) self.max_size = get_float(get_setting('max_size_episodes')) self.check_sizes() self.info = payload self.url = u"%s%s" % (definition['base_url'], show_query) if definition['tv_keywords']: self.queries = ["%s" % definition['tv_keywords']] self.extras = ["%s" % definition['tv_extra'] if definition['tv_extra'] else ''] # TODO this sucks, tv_keywords should be a list from the start.. if definition['tv_keywords2']: self.queries.append(definition['tv_keywords2']) self.extras.append(definition['tv_extra2'] if definition['tv_extra2'] else '')
def verify(self, provider, name, size): """ Main filtering method to match torrent names, resolutions, release types and size filters Args: provider (str): Provider ID name (str): Torrent name size (str): Arbitrary torrent size to be parsed Returns: bool: ``True`` if torrent name passed filtering, ``False`` otherwise. """ if not name: self.reason = '[%s] %s' % (provider, '*** Empty name ***') return False name = normalize_string(name) if self.filter_title and self.title: self.title = normalize_string(self.title) self.reason = "[%s] %70s ***" % (provider, name) if self.filter_resolutions and get_setting('require_resolution', bool): resolution = self.determine_resolution(name)[0] if resolution not in self.resolutions_allow: self.reason += " Resolution not allowed ({0})".format( resolution) return False if self.filter_title: if not all( map(lambda match: match in name, re.split( r'\s', self.title))): self.reason += " Name mismatch" return False if self.require_keywords and get_setting('require_keywords', bool): for required in self.require_keywords: if not self.included(name, keys=[required]): self.reason += " Missing required keyword" return False if not self.included_rx(name, keys=self.releases_allow) and get_setting( 'require_release_type', bool): self.reason += " Missing release type keyword" return False if self.included_rx(name, keys=self.releases_deny) and get_setting( 'require_release_type', bool): self.reason += " Blocked by release type keyword" return False if size and not self.in_size_range(size) and get_setting( 'require_size', bool): self.reason += " Size out of range ({0})".format(size) return False return True
def cleanup_results(results_list): """ Remove duplicate results, hash results without an info_hash, and sort by seeders Args: results_list (list): Results to clean-up Returns: list: De-duplicated, hashed and sorted results """ if len(results_list) == 0: return [] hashes = [] filtered_list = [] allow_noseeds = get_setting('allow_noseeds', bool) for result in results_list: if not result['seeds'] and not allow_noseeds: continue provider_name = result['provider'][result['provider'].find(']')+1:result['provider'].find('[/')] if not result['uri']: if not result['name']: continue try: log.warning('[%s] No URI for %s' % (provider_name, repr(result['name']))) except Exception as e: import traceback log.warning("%s logging failed with: %s" % (provider_name, repr(e))) map(log.debug, traceback.format_exc().split("\n")) continue hash_ = result['info_hash'].upper() if not hash_: if result['uri'] and result['uri'].startswith('magnet'): hash_ = Magnet(result['uri']).info_hash.upper() else: hash_ = hashlib.md5(result['uri']).hexdigest() try: log.debug("[%s] Hash for %s: %s" % (provider_name, repr(result['name']), hash_)) except Exception as e: import traceback log.warning("%s logging failed with: %s" % (result['provider'], repr(e))) map(log.debug, traceback.format_exc().split("\n")) if not any(existing == hash_ for existing in hashes): filtered_list.append(result) hashes.append(hash_) if (get_setting("sort_by_resolution", bool)): log.debug("[EXPEREMENTAL] Start last sorting list by resolution of all result before send to Elementum") filtered_list = sorted(filtered_list, key=lambda r: (get_int(r.pop('resolution'))), reverse=True) else: filtered_list = sorted(filtered_list, key=lambda r: (get_int(r['seeds'])), reverse=True) return filtered_list
def do_search(query, category=None): notice('Search query: ' + str(query)) try: partis = Partis(get_setting('username', unicode), get_setting('password', unicode)) notify('Searching ...') return partis.updateIconPath(partis.search(query, category), os.path.join(ADDON_PATH, 'Partis')) except Exception as e: log.debug(getattr(e, 'message', repr(e))) notify(getattr(e, 'message', repr(e))) return []
def got_results(provider, results): """ Results callback once a provider found all its results, or not Args: provider (str): The provider ID results (list): The list of results """ global provider_names global provider_results global available_providers definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) max_results = get_setting('max_results', int) sort_by = get_setting('sort_by', int) # 0 "Resolution" # 1 "Seeds" # 2 "Size" # 3 "Balanced" if not sort_by or sort_by == 3 or sort_by > 3: # TODO: think of something interesting to balance sort results sorted_results = sorted(results, key=lambda r: (nonesorter(r['sort_balance'])), reverse=True) elif sort_by == 0: sorted_results = sorted(results, key=lambda r: (nonesorter(r['sort_resolution'])), reverse=True) elif sort_by == 1: sorted_results = sorted(results, key=lambda r: (nonesorter(r['seeds'])), reverse=True) elif sort_by == 2: sorted_results = sorted(results, key=lambda r: (nonesorter(r['size'])), reverse=True) if len(sorted_results) > max_results: sorted_results = sorted_results[:max_results] log.info("[%s] >> %s returned %2d results in %.1f seconds%s" % (provider, definition['name'].rjust(longest), len(results), round(time.time() - request_time, 2), (", sending %d best ones" % max_results) if len(results) > max_results else "")) provider_results.extend(sorted_results) available_providers -= 1 if definition['name'] in provider_names: provider_names.remove(definition['name'])
def extract_subpage(q, name, torrent, size, seeds, peers, info_hash, referer): try: log.debug("[%s] Getting subpage at %s" % (provider, repr(torrent))) except Exception as e: import traceback log.error("[%s] Subpage logging failed with: %s" % (provider, repr(e))) map(log.debug, traceback.format_exc().split("\n")) # New client instance, otherwise it's race conditions all over the place subclient = Client() subclient.passkey = client.passkey headers = {} if get_setting("use_cloudhole", bool): subclient.clearance = get_setting('clearance') subclient.user_agent = get_setting('user_agent') if "subpage_mode" in definition: if definition["subpage_mode"] == "xhr": headers['X-Requested-With'] = 'XMLHttpRequest' headers['Content-Language'] = '' if referer: headers['Referer'] = referer uri = torrent.split('|') # Split cookies for private trackers subclient.open(uri[0].encode('utf-8'), headers=headers) if 'bittorrent' in subclient.headers.get('content-type', ''): log.debug('[%s] bittorrent content-type for %s' % (provider, repr(torrent))) if len(uri) > 1: # Stick back cookies if needed torrent = '%s|%s' % (torrent, uri[1]) else: try: torrent = extract_from_page(provider, subclient.content) if torrent and not torrent.startswith('magnet') and len( uri) > 1: # Stick back cookies if needed torrent = '%s|%s' % (torrent, uri[1]) except Exception as e: import traceback log.error( "[%s] Subpage extraction for %s failed with: %s" % (provider, repr(uri[0]), repr(e))) map(log.debug, traceback.format_exc().split("\n")) ret = (name, info_hash, torrent, size, seeds, peers) q.put_nowait(ret)
def generate_payload(provider, generator, filtering, verify_name=True, verify_size=True): """ Payload formatter to format results the way Elementum expects them Args: provider (str): Provider ID generator (function): Generator method, can be either ``extract_torrents`` or ``extract_from_api`` filtering (Filtering): Filtering class instance verify_name (bool): Whether to double-check the results' names match the query or not verify_size (bool): Whether to check the results' file sizes Returns: list: Formatted results """ filtering.information(provider) results = [] definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) for name, info_hash, uri, size, seeds, peers in generator: size = clean_size(size) # uri, info_hash = clean_magnet(uri, info_hash) v_name = name if verify_name else filtering.title v_size = size if verify_size else None if filtering.verify(provider, v_name, v_size): results.append({ "name": name, "uri": uri, "info_hash": info_hash, "size": size, "seeds": get_int(seeds), "peers": get_int(peers), "language": definition["language"] if 'language' in definition else 'en', "provider": '[COLOR %s]%s[/COLOR]' % (definition['color'], definition['name']), "icon": os.path.join(ADDON_PATH, 'burst', 'providers', 'icons', '%s.png' % provider), }) else: log.debug(filtering.reason.encode('utf-8')) log.debug('>>>>>> %s would send %d torrents to Elementum <<<<<<<' % (provider, len(results))) return results
def extract_from_page(provider, content): """ Sub-page extraction method Args: provider (str): Provider ID content (str): Page content from Client instance Returns: str: Torrent or magnet link extracted from sub-page """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if provider == "kinozal": matches = re.findall(r'\: (.{40})', content) # kinozal if matches: result = "magnet:?xt=urn:btih:" + matches[0] log.debug('[%s] Make magnet from info_hash: %s' % (provider, repr(result))) return result matches = re.findall(r'magnet:\?[^\'"\s<>\[\]]+', content) if matches: result = matches[0] log.debug('[%s] Matched magnet link: %s' % (provider, repr(result))) return result matches = re.findall('http(.*?).torrent["\']', content) if matches: result = 'http' + matches[0] + '.torrent' result = result.replace('torcache.net', 'itorrents.org') log.debug('[%s] Matched torrent link: %s' % (provider, repr(result))) return result matches = re.findall('/download\?token=[A-Za-z0-9%]+', content) if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched download link with token: %s' % (provider, repr(result))) return result matches = re.findall('/telechargement/[a-z0-9-_.]+', content) # cpasbien if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched some french link: %s' % (provider, repr(result))) return result matches = re.findall('/torrents/download/\?id=[a-z0-9-_.]+', content) # t411 if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched download link with an ID: %s' % (provider, repr(result))) return result return None
def __init__(self): self._counter = 0 self._cookies_filename = '' self._cookies = LWPCookieJar() self.user_agent = USER_AGENT self.clearance = None self.content = None self.status = None self.token = None self.passkey = None self.headers = dict() global dns_public_list global dns_opennic_list dns_public_list = get_setting("public_dns_list", unicode).replace(" ", "").split(",") dns_opennic_list = get_setting("opennic_dns_list", unicode).replace(" ", "").split(",") socket.setdefaulttimeout(60)
def cleanup_results(results_list): """ Remove duplicate results, hash results without an info_hash, and sort by seeders Args: results_list (list): Results to clean-up Returns: list: De-duplicated, hashed and sorted results """ if len(results_list) == 0: return [] hashes = [] filtered_list = [] allow_noseeds = get_setting('allow_noseeds', bool) for result in results_list: if not result['seeds'] and not allow_noseeds: continue if not result['uri']: if not result['name']: continue try: log.warning('[%s] No URI for %s' % (result['provider'][16:-8], repr(result['name']))) except Exception as e: import traceback log.warning("%s logging failed with: %s" % (result['provider'], repr(e))) map(log.debug, traceback.format_exc().split("\n")) continue hash_ = result['info_hash'].upper() if not hash_: try: if result['uri'] and result['uri'].startswith('magnet'): hash_ = Magnet(result['uri']).info_hash.upper() else: hash_ = hashlib.md5(py2_encode(result['uri'])).hexdigest() except: pass # try: # log.debug("[%s] Hash for %s: %s" % (result['provider'][16:-8], repr(result['name']), hash_)) # except Exception as e: # import traceback # log.warning("%s logging failed with: %s" % (result['provider'], repr(e))) # map(log.debug, traceback.format_exc().split("\n")) if not any(existing == hash_ for existing in hashes): filtered_list.append(result) hashes.append(hash_) return sorted(filtered_list, key=lambda r: (get_int(r['seeds'])), reverse=True)
def got_results(provider, results): """ Results callback once a provider found all its results, or not Args: provider (str): The provider ID results (list): The list of results """ global provider_names global provider_results global available_providers definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) max_results = get_setting('max_results', int) if get_setting('sort_by_resolution', bool): log.debug( "[%s][EXPEREMENTAL] Sorting by resolution before cutoff max_results" % provider) sorted_results = sorted(results, key=lambda r: (r['resolution']), reverse=True) else: sorted_results = sorted(results, key=lambda r: (r['seeds']), reverse=True) if get_setting('disable_max', bool): log.debug('[%s] Don\'t apply "max_results" settings' % provider) max_results = 999 elif len(sorted_results) > max_results: sorted_results = sorted_results[:max_results] log.info(">> %s returned %2d results in %.1f seconds%s" % (definition['name'].rjust(longest), len(results), round(time.time() - request_time, 2), (", sending %d best ones" % max_results) if len(results) > max_results else "")) provider_results.extend(sorted_results) available_providers -= 1 if definition['name'] in provider_names: provider_names.remove(definition['name'])
def use_general(self, provider, payload): """ Setup method to define general search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if get_setting("use_public_dns", bool) and "public_dns_alias" in definition: definition = get_alias(definition, definition["public_dns_alias"]) general_query = definition['general_query'] if definition[ 'general_query'] else '' log.debug("[%s] General URL: %s%s" % (provider, definition['base_url'], general_query)) self.info = payload self.url = u"%s%s" % (definition['base_url'], general_query) self.collect_queries('general', definition)
def use_anime(self, provider, info): """ Setup method to define anime search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) anime_query = definition['anime_query'] if definition['anime_query'] else '' log.debug("Anime URL: %s%s" % (definition['base_url'], anime_query)) if get_setting('separate_sizes', bool): self.min_size = get_float(get_setting('min_size_episodes')) self.max_size = get_float(get_setting('max_size_episodes')) self.check_sizes() self.info = info self.url = u"%s%s" % (definition['base_url'], anime_query) if self.info['absolute_number']: self.info['episode'] = self.info['absolute_number'] if definition['anime_keywords']: self.queries = ["%s" % definition['anime_keywords']] self.extras = ["%s" % definition['anime_extra'] if definition['anime_extra'] else '']
def get_enabled_providers(): """ Utility method to get all enabled provider IDs Returns: list: All available enabled provider IDs """ results = [] for provider in definitions: if get_setting('use_%s' % provider, bool): results.append(provider) if 'custom' in definitions[provider] and definitions[provider]['custom']: results.append(provider) return results
def extract_from_page(provider, content): """ Sub-page extraction method Args: provider (str): Provider ID content (str): Page content from Client instance Returns: str: Torrent or magnet link extracted from sub-page """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) matches = re.findall(r'magnet:\?[^\'"\s<>\[\]]+', content) if matches: result = matches[0] log.debug('[%s] Matched magnet link: %s' % (provider, repr(result))) return result matches = re.findall('http(.*?).torrent["\']', content) if matches: result = 'http' + matches[0] + '.torrent' result = result.replace('torcache.net', 'itorrents.org') log.debug('[%s] Matched torrent link: %s' % (provider, repr(result))) return result matches = re.findall('/download\?token=[A-Za-z0-9%]+', content) if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched download link with token: %s' % (provider, repr(result))) return result matches = re.findall('/torrents/download/\?id=[a-z0-9-_.]+', content) # t411 if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched download link with an ID: %s' % (provider, repr(result))) return result matches = re.findall( 'torrents\.php\?action=download&id=\d+&key=[a-fA-F0-9]+', content) # nCore if matches: result = definition['root_url'] + '/' + matches[0] log.debug('[%s] Matched download link with an ID: %s' % (provider, repr(result))) return result return None
def use_general(self, provider, payload): """ Setup method to define general search parameters Args: provider (str): Provider ID payload (dict): Elementum search payload """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) general_query = definition['general_query'] if definition['general_query'] else '' log.debug("General URL: %s%s" % (definition['base_url'], general_query)) self.info = payload self.url = u"%s%s" % (definition['base_url'], general_query) if definition['general_keywords']: self.queries = [definition['general_keywords']] self.extras = [definition['general_extra']]
def extract_from_page(provider, content): """ Sub-page extraction method Args: provider (str): Provider ID content (str): Page content from Client instance Returns: str: Torrent or magnet link extracted from sub-page """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) if provider == "kinozal": matches = re.findall(r'magnet:\?[^\'"\s<>\[\]]+', content) if matches: result = matches[0] log.debug('[%s] Matched magnet link: %s' % (provider, repr(result))) return result matches = re.findall(r'\: ([A-Fa-f0-9]{40})', content) # kinozal if matches: result = "magnet:?xt=urn:btih:" + matches[ 0] + "&tr=http%3A%2F%2Ftr0.torrent4me.com%2Fann%3Fuk%3Dstl41hKc1E&tr=http%3A%2F%2Ftr0.torrent4me.com%2Fann%3Fuk%3Dstl41hKc1E&tr=http%3A%2F%2Ftr0.tor4me.info%2Fann%3Fuk%3Dstl41hKc1E&tr=http%3A%2F%2Ftr0.tor2me.info%2Fann%3Fuk%3Dstl41hKc1E&tr=http%3A%2F%2Fretracker.local%2Fannounce" log.debug('[%s] Make magnet from info_hash: %s' % (provider, repr(result))) return result matches = re.findall(r'magnet:\?[^\'"\s<>\[\]]+', content) if matches: result = matches[0] log.debug('[%s] Matched magnet link: %s' % (provider, repr(result))) return result matches = re.findall('http(.*?).torrent["\']', content) if matches: result = 'http' + matches[0] + '.torrent' log.debug('[%s] Matched torrent link: %s' % (provider, repr(result))) return result matches = re.findall('"(/download/[A-Za-z0-9]+)"', content) if matches: result = definition['root_url'] + matches[0] log.debug('[%s] Matched download link: %s' % (provider, repr(result))) return result return None
def __init__(self): resolutions = OrderedDict() # TODO: remove when finished with debugging resolutions detection # resolutions['filter_240p'] = ['240p', u'240р', '_tvrip_', 'satrip', 'vhsrip'] # resolutions['filter_480p'] = ['480p', u'480р', 'xvid', 'dvd', 'dvdrip', 'hdtv'] # resolutions['filter_720p'] = ['720p', u'720р', 'hdrip', 'bluray', 'blu_ray', 'brrip', 'bdrip', 'hdtv', '/hd720p', '1280x720'] # resolutions['filter_1080p'] = ['1080p', u'1080р', '1080i', 'fullhd', '_fhd_', '/hd1080p', '/hdr1080p', '1920x1080'] # resolutions['filter_2k'] = ['_2k_', '1440p', u'1440р', u'_2к_'] # resolutions['filter_4k'] = ['_4k_', '2160p', u'2160р', '_uhd_', u'_4к_'] resolutions['filter_240p'] = [u'240[pр]', u'vhs\-?rip'] resolutions['filter_480p'] = [ u'480[pр]', u'xvid|dvd|dvdrip|hdtv|web\-(dl)?rip|iptv|sat\-?rip|tv\-?rip' ] resolutions['filter_720p'] = [ u'720[pр]|1280x720', u'hd720p?|hd\-?rip|b[rd]rip' ] resolutions['filter_1080p'] = [ u'1080[piр]|1920x1080', u'hd1080p?|fullhd|fhd|blu\W*ray|bd\W*remux' ] resolutions['filter_2k'] = [u'1440[pр]', u'2k'] resolutions['filter_4k'] = [u'4k|2160[pр]|uhd', u'4k|hd4k'] resolutions['filter_music'] = [u'mp3|flac|alac|ost|sound\-?track'] self.resolutions = resolutions self.release_types = { 'filter_brrip': [u'brrip|bd\-?rip|blu\-?ray|bd\-?remux'], 'filter_webdl': [u'web_?\-?dl|web\-?rip|dl\-?rip|yts'], 'filter_hdrip': [u'hd\-?rip'], 'filter_hdtv': [u'hd\-?tv'], 'filter_dvd': [u'dvd|dvd\-?rip|vcd\-?rip'], 'filter_dvdscr': [u'dvd\-?scr'], 'filter_screener': [u'screener|scr'], 'filter_3d': [u'3d'], 'filter_telesync': [u'telesync|ts|tc'], 'filter_cam': [u'cam|hd\-?cam'], 'filter_tvrip': [u'tv\-?rip|sat\-?rip|dvb'], 'filter_vhsrip': [u'vhs\-?rip'], 'filter_iptvrip': [u'iptv\-?rip'], 'filter_trailer': [u'trailer|трейлер|тизер'], 'filter_workprint': [u'workprint'], 'filter_line': [u'line'] } # TODO: remove when finished with debugging resolutions detection # self.release_types = { # 'filter_brrip': ['brrip', 'bdrip', 'bd-rip', 'bluray', 'blu-ray', 'bdremux', 'bd-remux'], # 'filter_webdl': ['webdl', 'webrip', 'web-rip', 'web_dl', 'dlrip', '_yts_'], # 'filter_hdrip': ['hdrip', 'hd-rip'], # 'filter_hdtv': ['hdtv'], # 'filter_dvd': ['_dvd_', 'dvdrip', 'dvd-rip', 'vcdrip'], # 'filter_dvdscr': ['dvdscr', 'dvd-scr'], # 'filter_screener': ['screener', '_scr_'], # 'filter_3d': ['_3d_'], # 'filter_telesync': ['telesync', '_ts_', '_tc_'], # 'filter_cam': ['_cam_', 'hdcam'], # 'filter_tvrip': ['_tvrip', 'satrip'], # 'filter_vhsrip': ['vhsrip'], # 'filter_trailer': ['trailer', u'трейлер', u'тизер'], # 'filter_workprint': ['workprint'] # } require = [] resolutions_allow = [] releases_allow = [] releases_deny = [] for resolution in self.resolutions: if get_setting(resolution, bool): resolutions_allow.append(resolution) # Add enabled resolutions to allowed release types to match # previous versions' behavior with certain providers # with no release types in torrent names, ie. YTS releases_allow.extend(self.resolutions[resolution]) self.resolutions_allow = resolutions_allow # Skip resolution filtering if we're allowing all of them anyway self.filter_resolutions = True if len(self.resolutions_allow) == len(self.resolutions): self.filter_resolutions = False for release_type in self.release_types: if get_setting(release_type, bool): releases_allow.extend(self.release_types[release_type]) else: releases_deny.extend(self.release_types[release_type]) self.releases_allow = releases_allow self.releases_deny = releases_deny if get_setting('additional_filters', bool): accept = get_setting('accept', unicode).strip().lower() if accept: accept = re.split(r',\s?', accept) releases_allow.extend(accept) block = get_setting('block', unicode).strip().lower() if block: block = re.split(r',\s?', block) releases_deny.extend(block) require = get_setting('require', unicode).strip().lower() if require: require = re.split(r',\s?', require) self.require_keywords = require self.min_size = get_float(get_setting('min_size')) self.max_size = get_float(get_setting('max_size')) self.check_sizes() self.filter_title = False self.queries = [] self.extras = [] self.info = dict(title="", proxy_url="", internal_proxy_url="", elementum_url="", titles=[]) self.kodi_language = '' self.language_exceptions = [] self.get_data = {} self.post_data = {} self.url = '' self.title = '' self.reason = '' self.results = []
def process_keywords(self, provider, text): """ Processes the query payload from a provider's keyword definitions Args: provider (str): Provider ID text (str): Keyword placeholders from definitions, ie. {title} Returns: str: Processed query keywords """ keywords = self.read_keywords(text) replacing = get_setting("filter_quotes", bool) for keyword in keywords: keyword = keyword.lower() if 'title' in keyword: title = self.info["title"] language = definitions[provider]['language'] use_language = None if ':' in keyword: use_language = keyword.split(':')[1].lower() if provider not in self.language_exceptions and \ (use_language or self.kodi_language) and \ 'titles' in self.info and self.info['titles']: try: if self.kodi_language and self.kodi_language in self.info[ 'titles']: use_language = self.kodi_language if use_language not in self.info['titles']: use_language = language if 'original' in self.info['titles']: title = self.info['titles']['original'] if use_language in self.info['titles'] and self.info[ 'titles'][use_language]: title = self.info['titles'][use_language] title = normalize_string(title) log.info("[%s] Using translated '%s' title %s" % (provider, use_language, repr(title))) log.debug( "[%s] Translated titles from Elementum: %s" % (provider, repr(self.info['titles']))) except Exception as e: import traceback log.error("%s failed with: %s" % (provider, repr(e))) map(log.debug, traceback.format_exc().split("\n")) text = text.replace('{%s}' % keyword, title) if 'year' in keyword: text = text.replace('{%s}' % keyword, str(self.info["year"])) if 'season' in keyword: if '+' in keyword: keys = keyword.split('+') season = str(self.info["season"] + get_int(keys[1])) elif ':' in keyword: keys = keyword.split(':') season = ('%%.%sd' % keys[1]) % self.info["season"] else: season = '%s' % self.info["season"] text = text.replace('{%s}' % keyword, season) if 'episode' in keyword: if '+' in keyword: keys = keyword.split('+') episode = str(self.info["episode"] + get_int(keys[1])) elif ':' in keyword: keys = keyword.split(':') episode = ('%%.%sd' % keys[1]) % self.info["episode"] else: episode = '%s' % self.info["episode"] text = text.replace('{%s}' % keyword, episode) if replacing: text = text.replace(u"'", '') return text
def extract_from_api(provider, client): """ Main API parsing generator for API-based providers An almost clever API parser, mostly just for YTS, RARBG and T411 Args: provider (str): Provider ID client (Client): Client class instance Yields: tuple: A torrent result """ try: data = json.loads(client.content) except: data = [] log.debug("[%s] JSON response from API: %s" % (unquote(provider), repr(data))) definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) api_format = definition['api_format'] results = [] result_keys = api_format['results'].split('.') log.debug("[%s] result_keys: %s" % (provider, repr(result_keys))) for key in result_keys: if key in data: data = data[key] else: data = [] # log.debug("[%s] nested results: %s" % (provider, repr(data))) results = data log.debug("[%s] results: %s" % (provider, repr(results))) if 'subresults' in api_format: from copy import deepcopy for result in results: # A little too specific to YTS but who cares... result['name'] = result[api_format['name']] subresults = [] subresults_keys = api_format['subresults'].split('.') for key in subresults_keys: for result in results: if key in result: for subresult in result[key]: sub = deepcopy(result) sub.update(subresult) subresults.append(sub) results = subresults log.debug("[%s] with subresults: %s" % (provider, repr(results))) for result in results: if not result or not isinstance(result, dict): continue name = '' info_hash = '' torrent = '' size = '' seeds = '' peers = '' if 'name' in api_format: name = result[api_format['name']] if 'description' in api_format: if name: name += ' ' name += result[api_format['description']] if 'torrent' in api_format: torrent = result[api_format['torrent']] if 'download_path' in definition: torrent = definition['base_url'] + definition[ 'download_path'] + torrent if client.token: user_agent = USER_AGENT headers = { 'Authorization': client.token, 'User-Agent': user_agent } log.debug("[%s] Appending headers: %s" % (provider, repr(headers))) torrent = append_headers(torrent, headers) log.debug("[%s] Torrent with headers: %s" % (provider, repr(torrent))) if 'info_hash' in api_format: info_hash = result[api_format['info_hash']] if 'quality' in api_format: # Again quite specific to YTS... name = "%s - %s" % (name, result[api_format['quality']]) if 'size' in api_format: size = result[api_format['size']] if type(size) in (long, int): size = sizeof(size) elif type(size) in (str, unicode) and size.isdigit(): size = sizeof(int(size)) if 'seeds' in api_format: seeds = result[api_format['seeds']] if type(seeds) in (str, unicode) and seeds.isdigit(): seeds = int(seeds) if 'peers' in api_format: peers = result[api_format['peers']] if type(peers) in (str, unicode) and peers.isdigit(): peers = int(peers) yield (name, info_hash, torrent, size, seeds, peers)
def search(payload, method="general"): """ Main search entrypoint Args: payload (dict): Search payload from Elementum. method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime`` Returns: list: All filtered results in the format Elementum expects """ log.debug("Searching with payload (%s): %s" % (method, repr(payload))) if method == 'general': if 'query' in payload: payload['title'] = payload['query'] payload['titles'] = {'source': payload['query']} else: payload = { 'title': payload, 'titles': { 'source': payload }, } payload['titles'] = dict( (k.lower(), v) for k, v in payload['titles'].iteritems()) # If titles[] exists in payload and there are special chars in titles[source] # then we set a flag to possibly modify the search query payload['has_special'] = 'titles' in payload and \ bool(payload['titles']) and \ 'source' in payload['titles'] and \ any(c in payload['titles']['source'] for c in special_chars) if payload['has_special']: log.debug( "Query title contains special chars, so removing any quotes in the search query" ) if 'proxy_url' not in payload: payload['proxy_url'] = '' if 'internal_proxy_url' not in payload: payload['internal_proxy_url'] = '' if 'elementum_url' not in payload: payload['elementum_url'] = '' if 'silent' not in payload: payload['silent'] = False if 'skip_auth' not in payload: payload['skip_auth'] = False global request_time global provider_names global provider_results global available_providers provider_names = [] provider_results = [] available_providers = 0 request_time = time.time() providers = get_enabled_providers(method) if len(providers) == 0: if not payload['silent']: notify(translation(32060), image=get_icon_path()) log.error("No providers enabled") return [] log.info( "Burstin' with %s" % ", ".join([definitions[provider]['name'] for provider in providers])) if get_setting('kodi_language', bool): kodi_language = xbmc.getLanguage(xbmc.ISO_639_1) if not kodi_language: log.warning("Kodi returned empty language code...") elif 'titles' not in payload or not payload['titles']: log.info("No translations available...") elif payload['titles'] and kodi_language not in payload['titles']: log.info("No '%s' translation available..." % kodi_language) p_dialog = xbmcgui.DialogProgressBG() if not payload['silent']: p_dialog.create('Elementum [COLOR FFFF6B00]Burst[/COLOR]', translation(32061)) for provider in providers: available_providers += 1 provider_names.append(definitions[provider]['name']) task = Thread(target=run_provider, args=(provider, payload, method)) task.start() providers_time = time.time() total = float(available_providers) # Exit if all providers have returned results or timeout reached, check every 100ms while time.time() - providers_time < timeout and available_providers > 0: timer = time.time() - providers_time log.debug("Timer: %ds / %ds" % (timer, timeout)) if timer > timeout: break message = translation( 32062 ) % available_providers if available_providers > 1 else translation( 32063) if not payload['silent']: p_dialog.update(int((total - available_providers) / total * 100), message=message) time.sleep(0.25) if not payload['silent']: p_dialog.close() del p_dialog if available_providers > 0: message = u', '.join(provider_names) message = message + translation(32064) log.warning(message.encode('utf-8')) if not payload['silent']: notify(message, ADDON_ICON) log.debug("all provider_results: %s" % repr(provider_results)) filtered_results = apply_filters(provider_results) log.debug("all filtered_results: %s" % repr(filtered_results)) log.info("Providers returned %d results in %s seconds" % (len(filtered_results), round(time.time() - request_time, 2))) return filtered_results
def extract_torrents(provider, client): """ Main torrent extraction generator for non-API based providers Args: provider (str): Provider ID client (Client): Client class instance Yields: tuple: A torrent result """ definition = definitions[provider] definition = get_alias(definition, get_setting("%s_alias" % provider)) log.debug("[%s] Extracting torrents from %s using definitions: %s" % (provider, provider, repr(definition))) if not client.content: if get_setting("use_debug_parser", bool): log.debug("[%s] Parser debug | Page content is empty" % provider) raise StopIteration dom = Html().feed(client.content) key_search = get_search_query(definition, "key") row_search = get_search_query(definition, "row") name_search = get_search_query(definition, "name") torrent_search = get_search_query(definition, "torrent") info_hash_search = get_search_query(definition, "infohash") size_search = get_search_query(definition, "size") seeds_search = get_search_query(definition, "seeds") peers_search = get_search_query(definition, "peers") referer_search = get_search_query(definition, "referer") log.debug("[%s] Parser: %s" % (provider, repr(definition['parser']))) q = Queue() threads = [] needs_subpage = 'subpage' in definition and definition['subpage'] if needs_subpage: def extract_subpage(q, name, torrent, size, seeds, peers, info_hash, referer): try: log.debug("[%s] Getting subpage at %s" % (provider, repr(torrent))) except Exception as e: import traceback log.error("[%s] Subpage logging failed with: %s" % (provider, repr(e))) map(log.debug, traceback.format_exc().split("\n")) # New client instance, otherwise it's race conditions all over the place subclient = Client() subclient.passkey = client.passkey headers = {} if "subpage_mode" in definition: if definition["subpage_mode"] == "xhr": headers['X-Requested-With'] = 'XMLHttpRequest' headers['Content-Language'] = '' if referer: headers['Referer'] = referer uri = torrent.split('|') # Split cookies for private trackers subclient.open(uri[0].encode('utf-8'), headers=headers) if 'bittorrent' in subclient.headers.get('content-type', ''): log.debug('[%s] bittorrent content-type for %s' % (provider, repr(torrent))) if len(uri) > 1: # Stick back cookies if needed torrent = '%s|%s' % (torrent, uri[1]) else: try: torrent = extract_from_page(provider, subclient.content) if torrent and not torrent.startswith('magnet') and len( uri) > 1: # Stick back cookies if needed torrent = '%s|%s' % (torrent, uri[1]) except Exception as e: import traceback log.error( "[%s] Subpage extraction for %s failed with: %s" % (provider, repr(uri[0]), repr(e))) map(log.debug, traceback.format_exc().split("\n")) ret = (name, info_hash, torrent, size, seeds, peers) q.put_nowait(ret) if not dom: if get_setting("use_debug_parser", bool): log.debug( "[%s] Parser debug | Could not parse DOM from page content" % provider) raise StopIteration if get_setting("use_debug_parser", bool): log.debug( "[%s] Parser debug | Page content: %s" % (provider, client.content.replace('\r', '').replace('\n', ''))) key = eval(key_search) if key_search else "" if key_search and get_setting("use_debug_parser", bool): key_str = key.__str__() log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'key', key_search, key_str.replace('\r', '').replace( '\n', ''))) items = eval(row_search) if get_setting("use_debug_parser", bool): log.debug("[%s] Parser debug | Matched %d items for '%s' query '%s'" % (provider, len(items), 'row', row_search)) for item in items: if get_setting("use_debug_parser", bool): item_str = item.__str__() log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'row', row_search, item_str.replace( '\r', '').replace('\n', ''))) if not item: continue try: name = eval(name_search) if name_search else "" torrent = eval(torrent_search) if torrent_search else "" size = eval(size_search) if size_search else "" seeds = eval(seeds_search) if seeds_search else "" peers = eval(peers_search) if peers_search else "" info_hash = eval(info_hash_search) if info_hash_search else "" referer = eval(referer_search) if referer_search else "" if 'magnet:?' in torrent: torrent = torrent[torrent.find('magnet:?'):] if get_setting("use_debug_parser", bool): log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'name', name_search, name)) log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'torrent', torrent_search, torrent)) log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'size', size_search, size)) log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'seeds', seeds_search, seeds)) log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'peers', peers_search, peers)) if info_hash_search: log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'info_hash', info_hash_search, info_hash)) if referer_search: log.debug( "[%s] Parser debug | Matched '%s' iteration for query '%s': %s" % (provider, 'info_hash', referer_search, referer)) # Pass client cookies with torrent if private if not torrent.startswith('magnet'): user_agent = USER_AGENT if client.passkey: torrent = torrent.replace('PASSKEY', client.passkey) elif client.token: headers = { 'Authorization': client.token, 'User-Agent': user_agent } log.debug("[%s] Appending headers: %s" % (provider, repr(headers))) torrent = append_headers(torrent, headers) log.debug("[%s] Torrent with headers: %s" % (provider, repr(torrent))) else: parsed_url = urlparse(torrent.split('|')[0]) cookie_domain = '{uri.netloc}'.format(uri=parsed_url) cookie_domain = re.sub('www\d*\.', '', cookie_domain) cookies = [] for cookie in client._cookies: if cookie_domain in cookie.domain: cookies.append(cookie) headers = {} if cookies: headers = {'User-Agent': user_agent} log.debug("[%s] Cookies res: %s / %s" % (provider, repr(headers), repr(client.request_headers))) if client.request_headers: headers.update(client.request_headers) if client.url: headers['Referer'] = client.url headers['Origin'] = client.url # Need to set Cookie afterwards to avoid rewriting it with session Cookies headers['Cookie'] = ";".join( ["%s=%s" % (c.name, c.value) for c in cookies]) else: headers = {'User-Agent': user_agent} torrent = append_headers(torrent, headers) if name and torrent and needs_subpage and not torrent.startswith( 'magnet'): if not torrent.startswith('http'): torrent = definition['root_url'] + torrent.encode('utf-8') t = Thread(target=extract_subpage, args=(q, name, torrent, size, seeds, peers, info_hash, referer)) threads.append(t) else: yield (name, info_hash, torrent, size, seeds, peers) except Exception as e: log.error("[%s] Got an exception while parsing results: %s" % (provider, repr(e))) if needs_subpage: log.debug("[%s] Starting subpage threads..." % provider) for t in threads: t.start() for t in threads: t.join() for i in range(q.qsize()): ret = q.get_nowait() log.debug("[%s] Queue %d got: %s" % (provider, i, repr(ret))) yield ret