def results(self, info): try: self.info = info self.db_type = self.info.get("db_type") self.title = self.info.get("title") self.year = self.info.get("year") if self.year: self.rootname = '%s (%s)' % (self.title, self.year) else: self.rootname = self.title self.season = self.info.get("season", None) self.episode = self.info.get("episode", None) self.extensions = supported_video_extensions() self.folder_query = clean_title(normalize(self.title)) self.file_query = clean_title(normalize(self.title)) self.query_list = self._year_query_list( ) if self.db_type == 'movie' else self._episode_query_list() self._scrape_cloud() if not self.scrape_results: return self.sources self.label_settings = label_settings(self.info['scraper_settings'], self.scrape_provider) for item in self.scrape_results: try: file_name = normalize(item['name']) file_dl = item['url_link'] size = float(int(item['size'])) / 1073741824 video_quality = get_release_quality(file_name) details = get_file_info(file_name) label, multiline_label = build_internal_scrapers_label( self.label_settings, file_name, details, size, video_quality) self.sources.append({ 'name': file_name, 'label': label, 'multiline_label': multiline_label, 'title': file_name, 'quality': video_quality, 'size': size, 'url_dl': file_dl, 'id': file_dl, 'downloads': False, 'direct': True, 'source': self.scrape_provider, 'scrape_provider': self.scrape_provider }) except: pass window.setProperty('ad-cloud_source_results', json.dumps(self.sources)) except Exception as e: from modules.utils import logger logger('FEN alldebrid scraper Exception', e) return self.sources
def _scrape_downloads(self): try: my_downloads = RealDebrid.downloads() my_downloads = [ i for i in my_downloads if i['download'].lower().endswith(tuple(self.extensions)) ] append = self.scrape_results.append for item in my_downloads: match = False normalized = normalize(item['filename']) filename = clean_title(normalized) if self.db_type == 'movie': if any(x in filename for x in self._year_query_list()): match = True else: if seas_ep_filter(self.season, self.episode, normalized): match = True if match and self.folder_query in filename: item = self.make_downloads_item(item) if item['path'] not in [ d['path'] for d in self.scrape_results ]: append(item) except: pass
def _scrape_directory(self, folder_name): folder_files = [] folder_results = [] dirs, files = xbmcvfs.listdir(folder_name) for i in dirs: folder_files.append((i, 'folder')) for i in files: folder_files.append((i, 'file')) for item in folder_files: file_type = item[1] item_name = clean_title(normalize(item[0])) if file_type == 'file': ext = os.path.splitext(urlparse(item[0]).path)[-1] if ext in self.extensions: if self.db_type == 'movie': if self.title_query in item_name: url_path = self.url_path(folder_name, item[0]) self.scrape_results.append((item[0], url_path)) elif any(x in item_name for x in self.file_query): if not folder_name in self.folder_path: url_path = self.url_path(folder_name, item[0]) self.scrape_results.append((item[0], url_path)) elif self.title_query in item_name: url_path = self.url_path(folder_name, item[0]) self.scrape_results.append((item[0], url_path)) elif file_type == 'folder': if self.title_query in item_name or any( x in item_name for x in self.folder_query): new_folder = os.path.join(folder_name, item[0]) folder_results.append(new_folder) if not folder_results: return return self._scraper_worker(folder_results)
def _scrape_directory(self, folder_name): folder_results = [] dirs, files = xbmcvfs.listdir(folder_name) cloud_files = files + dirs for item in cloud_files: ext = os.path.splitext(urlparse(item).path)[-1] file_type = 'file' if ext else 'folder' item_name = clean_title(normalize(item)) if file_type == 'file' and ext in self.extensions: if self.db_type == 'movie': if self.title_query in item_name: url_path = self.url_path(folder_name, item) self.scrape_results.append((item, url_path)) elif any(x in item_name for x in self.file_query): if not folder_name == self.download_path: url_path = self.url_path(folder_name, item) self.scrape_results.append((item, url_path)) elif self.title_query in item_name: url_path = self.url_path(folder_name, item) self.scrape_results.append((item, url_path)) elif file_type == 'folder': if self.title_query in item_name or any( x in item_name for x in self.folder_query): new_folder = os.path.join(folder_name, item) folder_results.append(new_folder) if not folder_results: return return self._scraper_worker(folder_results)
def _scrape_cloud(self): try: threads = [] cloud_files = [] try: my_cloud_files = AllDebrid.user_cloud() except: return self.sources try: for k, v in my_cloud_files.iteritems(): if isinstance(v, dict): cloud_files.append(v) except: for k, v in my_cloud_files.items(): if isinstance(v, dict): cloud_files.append(v) my_cloud_files = [i for i in cloud_files if i['statusCode'] == 4] for item in my_cloud_files: folder_name = clean_title(normalize(item['filename'])) assigned_content = self._assigned_content( normalize(item['filename'])) if assigned_content: if assigned_content == self.rootname: self.folder_results.append( (normalize(item['filename']), item, True)) elif self.folder_query in folder_name or not folder_name: self.folder_results.append( (normalize(item['filename']), item, False)) if not self.folder_results: return self.sources for i in self.folder_results: threads.append(Thread(target=self._scrape_folders, args=(i, ))) [i.start() for i in threads] [i.join() for i in threads] except: pass
def _scrape_cloud(self): try: try: my_cloud_files = RealDebrid.user_cloud() except: return self.sources append = self.folder_results.append for item in my_cloud_files: normalized = normalize(item['filename']) folder_name = clean_title(normalized) assigned_content = self._assigned_content(normalized) if assigned_content: if assigned_content == self.rootname: append((normalized, item['id'], True)) elif self.folder_query in folder_name or not folder_name: append((normalized, item['id'], False)) if not self.folder_results: return self.sources threads = [] append = threads.append for i in self.folder_results: append(Thread(target=self._scrape_folders, args=(i, ))) [i.start() for i in threads] [i.join() for i in threads] except: pass
def rd_external_browser(magnet, filtering_list): import re try: from HTMLParser import HTMLParser except ImportError: from html.parser import HTMLParser try: episode_match = False torrent_id = None torrent_keys = [] extensions = supported_video_extensions() magnet_url = HTMLParser().unescape(magnet) r = re.search('''magnet:.+?urn:([a-zA-Z0-9]+):([a-zA-Z0-9]+)''', str(magnet), re.I) infoHash = r.group(2).lower() torrent_files = RealDebrid.check_hash(infoHash) torrent_files = torrent_files[infoHash]['rd'][0] try: files_tuple = sorted([ (k, v['filename'].lower()) for k, v in torrent_files.items() if v['filename'].lower().endswith(tuple(extensions)) ]) except: return None files_tuple = sorted(files_tuple) for i in files_tuple: if any(x in i[1] for x in filtering_list): episode_match = True torrent_keys.append(i[0]) if not episode_match: return None if not torrent_keys: return None torrent_keys = ','.join(torrent_keys) torrent = RealDebrid.add_magnet(magnet_url) torrent_id = torrent['id'] RealDebrid.add_torrent_select(torrent_id, torrent_keys) torrent_files = RealDebrid.user_cloud_info(torrent_id) file_info = [ i for i in torrent_files['files'] if i['path'].lower().endswith(tuple(extensions)) ] file_urls = torrent_files['links'] pack_info = [ dict(i.items() + [('url_link', file_urls[c])]) for c, i in enumerate(file_info) ] pack_info = sorted(pack_info, key=lambda k: k['path']) for item in pack_info: filename = clean_title(item['path']) if any(x in filename for x in filtering_list): correct_result = item break url_link = correct_result['url_link'] RealDebrid.delete_torrent(torrent_id) return resolve_rd(url_link, play=False) except: if torrent_id: RealDebrid.delete_torrent(torrent_id) return None
def _scrape_history(self): try: my_downloads = RealDebrid.downloads() my_downloads = [i for i in my_downloads if i['download'].lower().endswith(tuple(self.extensions))] for item in my_downloads: filename = clean_title(normalize(item['filename'])) if any(x in filename for x in self.query_list): if self.folder_query in filename: item = self.make_history_item(item) if item['path'] not in [d['path'] for d in self.scrape_results]: self.scrape_results.append(item) except: pass
def get_filename_match(title, url, name=None): from modules.utils import clean_file_name if name: return clean_file_name(name) from modules.utils import clean_title, normalize title_match = None try: title = clean_title(normalize(title)) name_url = unquote(url) try: file_name = clean_title(name_url.split('/')[-1]) except: return title_match test = name_url.split('/') for item in test: test_url = string(clean_title(normalize(item))) if title in test_url: title_match = clean_file_name(string(item)).replace( 'html', ' ').replace('+', ' ') break except: pass return title_match
def getFilename(self): if self.final_name: final_name = self.final_name elif self.action == 'meta.pack': name = self.params_get('pack_files')['filename'] final_name = os.path.splitext( urlparse(name).path)[0].split('/')[-1] elif self.action == 'image': final_name = self.title else: name_url = unquote(self.url) file_name = clean_title(name_url.split('/')[-1]) if clean_title(self.title).lower() in file_name.lower(): final_name = os.path.splitext( urlparse(name_url).path)[0].split('/')[-1] else: try: final_name = self.name.translate(None, r'\/:*?"<>|').strip('.') except: final_name = os.path.splitext( urlparse(name_url).path)[0].split('/')[-1] self.final_name = to_utf8(safe_string(remove_accents(final_name)))
def results(self, info): try: self.info = info self.db_type = self.info.get("db_type") self.download_path = settings.download_directory(self.db_type) self.title = self.info.get("title") self.year = self.info.get("year") self.season = self.info.get("season") self.episode = self.info.get("episode") self.title_query = clean_title(self.title) self.folder_query = self._season_query_list( ) if self.db_type == 'episode' else self._year_query_list() self.file_query = self._episode_query_list( ) if self.db_type == 'episode' else self._year_query_list() self._scrape_directory(self.download_path) if not self.scrape_results: return self.sources self.label_settings = label_settings(self.info['scraper_settings'], self.scrape_provider) for item in self.scrape_results: try: file_name = item[0] file_dl = item[1] size = self._get_size(file_dl) details = get_file_info(file_name) video_quality = get_release_quality(file_name, file_dl) label, multiline_label = build_internal_scrapers_label( self.label_settings, file_name, details, size, video_quality) self.sources.append({ 'name': file_name, 'label': label, 'multiline_label': multiline_label, 'title': file_name, 'quality': video_quality, 'size': size, 'url_dl': file_dl, 'id': file_dl, 'downloads': True, 'direct': True, 'source': self.scrape_provider, 'scrape_provider': self.scrape_provider }) except: pass window.setProperty('downloads_source_results', json.dumps(self.sources)) except Exception as e: from modules.utils import logger logger('FEN downloads scraper Exception', e) return self.sources
def getFileNameMatch(title, url): try: from urllib import unquote except ImportError: from urllib.parse import unquote from modules.utils import clean_title, normalize, clean_file_name title_match = None try: title = clean_title(normalize(title)) if 'magnet' in url: url = url.split('&dn=')[1] name_url = unquote(url) try: file_name = clean_title(name_url.split('/')[-1]) except: return title_match test = name_url.split('/') for item in test: test_url = str(clean_title(normalize(item))) if title in test_url: title_match = clean_file_name(str(item)).replace( '&tr=udp:', '').replace('&tr=http:', '').replace( '&tr=udp', '').replace('&tr=http', '').replace('html', ' ').replace('+', ' ') try: title_match = title_match.split('&xl=')[0] except: pass try: title_match = title_match.split(' p=')[0] except: pass break except: pass return title_match
def _scrape_cloud(self): try: cloud_files = Premiumize.user_cloud_all()['files'] cloud_files = [ i for i in cloud_files if i['path'].lower().endswith(tuple(self.extensions)) ] cloud_files = sorted(cloud_files, key=lambda k: k['name']) except: return self.sources for item in cloud_files: item_name = clean_title(normalize(item['name'])) if self.query in item_name: if any(x in item_name for x in self.file_query): self.scrape_results.append(item)
def _scrape_folders(self, folder_info): try: assigned_folder = folder_info[2] folder_files = RealDebrid.user_cloud_info(folder_info[1]) file_info = [i for i in folder_files['files'] if i['path'].lower().endswith(tuple(self.extensions))] file_urls = folder_files['links'] for c, i in enumerate(file_info): try: i.update({'folder_name': folder_info[0], 'url_link': file_urls[c]}) except: pass contents = sorted(file_info, key=lambda k: k['path']) for item in contents: filename = clean_title(normalize(item['path'])) if any(x in filename for x in self.query_list): if assigned_folder: self.scrape_results.append(item) elif self.folder_query in filename: self.scrape_results.append(item) except: pass
def _scrape_folders(self, folder_info): try: final_files = [] extensions = supported_video_extensions() assigned_folder = folder_info[2] torrent_folder = folder_info[1] links = torrent_folder['links'] total_size = torrent_folder['size'] try: links_count = len([ v for k, v in links.items() if v.lower().endswith(tuple(self.extensions)) ]) for k, v in links.items(): if v.lower().endswith(tuple(self.extensions)): size = total_size / links_count final_files.append({ 'name': v, 'url_link': k, 'size': size }) except: links_count = len([ v for k, v in links.iteritems() if v.lower().endswith(tuple(self.extensions)) ]) for k, v in links.iteritems(): if v.lower().endswith(tuple(self.extensions)): size = total_size / links_count final_files.append({ 'name': v, 'url_link': k, 'size': size }) for item in final_files: filename = clean_title(normalize(item['name'])) if any(x in filename for x in self.query_list): if assigned_folder: self.scrape_results.append(item) elif self.folder_query in filename: self.scrape_results.append(item) except: return
def _scrape_folders(self, folder_info): try: assigned_folder = folder_info[2] folder_files = RealDebrid.user_cloud_info(folder_info[1]) contents = [ i for i in folder_files['files'] if i['path'].lower().endswith(tuple(self.extensions)) ] file_urls = folder_files['links'] append = self.scrape_results.append for c, i in enumerate(contents): try: i.update({ 'folder_name': folder_info[0], 'url_link': file_urls[c] }) except: pass contents.sort(key=lambda k: k['path']) for item in contents: match = False normalized = normalize(item['path']) filename = clean_title(normalized) if assigned_folder and self.db_type == 'movie': match = True else: if self.db_type == 'movie': if any(x in filename for x in self._year_query_list() ) and self.folder_query in filename: match = True else: if assigned_folder: if any(x in normalized.lower() for x in self.seas_ep_query_list): match = True elif seas_ep_filter(self.season, self.episode, normalized): match = True if match: if assigned_folder: item['assigned_folder'] = True append(item) except: pass
def _scrape_cloud(self): try: cloud_files = Premiumize.user_cloud_all()['files'] cloud_files = [ i for i in cloud_files if i['path'].lower().endswith(tuple(self.extensions)) ] cloud_files.sort(key=lambda k: k['name']) except: return self.sources append = self.scrape_results.append for item in cloud_files: normalized = normalize(item['name']) item_name = clean_title(normalized) if self.query in item_name: if self.db_type == 'movie': if any(x in item['name'] for x in self._year_query_list()): append(item) elif seas_ep_filter(self.season, self.episode, normalized): append(item)
def download(url): from modules.nav_utils import hide_busy_dialog, notification from modules.utils import clean_file_name, clean_title from modules import settings # from modules.utils import logger if url == None: hide_busy_dialog() notification('No URL found for Download. Pick another Source', 6000) return params = dict(parse_qsl(sys.argv[2].replace('?',''))) json_meta = params.get('meta') if json_meta: meta = json.loads(json_meta) db_type = meta.get('vid_type') title = meta.get('search_title') year = meta.get('year') image = meta.get('poster') season = meta.get('season') episode = meta.get('episode') name = params.get('name') else: db_type = params.get('db_type') image = params.get('image') title = params.get('name') title = clean_file_name(title) media_folder = settings.download_directory(db_type) if not media_folder: hide_busy_dialog() resp = xbmcgui.Dialog().yesno( "No Download folder set!", "Fen requires you to set Download Folders.", "Would you like to set a folder now?") if resp: from modules.nav_utils import open_settings return open_settings('7.0') else: return if db_type in ('movie', 'episode'): folder_rootname = '%s (%s)' % (title, year) folder = os.path.join(media_folder, folder_rootname + '/') else: folder = media_folder if db_type == 'episode': folder = os.path.join(folder, 'Season ' + str(season)) try: headers = dict(parse_qsl(url.rsplit('|', 1)[1])) except: headers = dict('') dest = None url = url.split('|')[0] if not 'http' in url: from apis.furk_api import FurkAPI from indexers.furk import filter_furk_tlist, seas_ep_query_list t_files = FurkAPI().t_files(url) t_files = [i for i in t_files if 'video' in i['ct'] and 'bitrate' in i] name, url = filter_furk_tlist(t_files, (None if db_type == 'movie' else seas_ep_query_list(season, episode)))[0:2] dest = os.path.join(folder, name) if db_type == 'archive': ext = 'zip' if db_type == 'audio': ext = os.path.splitext(urlparse(url).path)[1][1:] if not ext in ['wav', 'mp3', 'ogg', 'flac', 'wma', 'aac']: ext = 'mp3' else: ext = os.path.splitext(urlparse(url).path)[1][1:] if not ext in ['mp4', 'mkv', 'flv', 'avi', 'mpg']: ext = 'mp4' if not dest: name_url = unquote(url) file_name = clean_title(name_url.split('/')[-1]) if clean_title(title).lower() in file_name.lower(): transname = os.path.splitext(urlparse(name_url).path)[0].split('/')[-1] else: try: transname = name.translate(None, '\/:*?"<>|').strip('.') except: transname = os.path.splitext(urlparse(name_url).path)[0].split('/')[-1] dest = os.path.join(folder, transname + '.' + ext) sysheaders = quote_plus(json.dumps(headers)) sysurl = quote_plus(url) systitle = quote_plus(transname) sysimage = quote_plus(image) sysfolder = quote_plus(folder) sysdest = quote_plus(dest) script = inspect.getfile(inspect.currentframe()) cmd = 'RunScript(%s, %s, %s, %s, %s, %s, %s)' % (script, sysurl, sysdest, systitle, sysimage, sysfolder, sysheaders) xbmc.executebuiltin(cmd)
def filter_furk_tlist(t_files, filtering_list=None): from modules.utils import clean_title, normalize t_files = [i for i in t_files if 'video' in i['ct'] and any(x in clean_title(normalize(i['name'])) for x in filtering_list) and not any(x in i['name'].lower() for x in ['furk320', 'sample'])][0] if filtering_list else [i for i in t_files if 'is_largest' in i][0] return t_files['name'], t_files['url_dl'], t_files['size']
def results(self, info): try: if not enabled_debrids_check('pm'): return internal_results(self.scrape_provider, self.sources) self.title_filter = filter_by_name(self.scrape_provider) self.sources, self.scrape_results = [], [] self.db_type = info.get('db_type') self.title = info.get('title') self.year = info.get('year') if self.year: self.rootname = '%s (%s)' % (self.title, self.year) else: self.rootname = self.title self.season = info.get('season') self.episode = info.get('episode') self.query = clean_title(self.title) self.extensions = supported_video_extensions() self._scrape_cloud() if not self.scrape_results: return internal_results(self.scrape_provider, self.sources) self.aliases = get_aliases_titles(info.get('aliases', [])) def _process(): for item in self.scrape_results: try: file_name = normalize(item['name']) if self.title_filter: if not check_title(self.title, file_name, self.aliases, self.year, self.season, self.episode): continue URLName = clean_file_name(file_name).replace( 'html', ' ').replace('+', ' ').replace('-', ' ') path = item['path'] file_dl = item['id'] size = round(float(item['size']) / 1073741824, 2) video_quality, details = get_file_info( name_info=release_info_format(file_name)) source_item = { 'name': file_name, 'title': file_name, 'URLName': URLName, 'quality': video_quality, 'size': size, 'size_label': '%.2f GB' % size, 'extraInfo': details, 'url_dl': file_dl, 'id': file_dl, 'downloads': False, 'direct': True, 'source': self.scrape_provider, 'scrape_provider': self.scrape_provider } yield source_item except: pass self.sources = list(_process()) except Exception as e: from modules.kodi_utils import logger logger('FEN premiumize scraper Exception', e) internal_results(self.scrape_provider, self.sources) return self.sources
def results(self, info): try: self.info = info self.db_type = self.info.get("db_type") self.folder_path = settings.source_folders_directory( self.db_type, self.scrape_provider) if not self.folder_path: return self.sources self.title = self.info.get("title") self.year = self.info.get("year") self.season = self.info.get("season") self.episode = self.info.get("episode") self.title_query = clean_title(self.title) self.folder_query = self._season_query_list( ) if self.db_type == 'episode' else self._year_query_list() self.file_query = self._episode_query_list( ) if self.db_type == 'episode' else self._year_query_list() cache_name = 'fen_%s_SCRAPER_%s_%s_%s_%s' % ( self.scrape_provider, self.title, self.year, self.season, self.episode) cache = _cache.get(cache_name) if cache: self.scrape_results = cache else: self._scrape_directory(self.folder_path) _cache.set(cache_name, self.scrape_results, expiration=datetime.timedelta(hours=2)) if not self.scrape_results: return self.sources self.label_settings = label_settings(self.info['scraper_settings'], 'folders', self.scraper_name) for item in self.scrape_results: try: file_name = item[0] file_dl = item[1] size = self._get_size( file_dl) if not file_dl.endswith('.strm') else 'strm' details = get_file_info(file_name) video_quality = get_release_quality(file_name, file_dl) label, multiline_label = build_internal_scrapers_label( self.label_settings, file_name, details, size, video_quality) self.sources.append({ 'name': file_name, 'label': label, 'multiline_label': multiline_label, 'title': file_name, 'quality': video_quality, 'size': size, 'url_dl': file_dl, 'id': file_dl, self.scrape_provider: True, 'direct': True, 'source': self.scrape_provider, 'scrape_provider': self.scrape_provider }) except: pass window.setProperty('%s_source_results' % self.scrape_provider, json.dumps(self.sources)) except Exception as e: from modules.utils import logger logger('FEN folders scraper Exception', e) return self.sources