def get_release_quality(release_title, release_link=None): try: quality = None fmt = release_title_strip(release_title) if fmt is not None: quality = get_qual(fmt) if not quality: if release_link: release_link = release_link.lower() try: release_link = release_link.encode('utf-8') except: pass quality = get_qual(release_link) if not quality: quality = 'SD' else: quality = 'SD' info = [] if fmt is not None: if any(value in fmt for value in VIDEO_3D): info.append('3D') if any(value in fmt for value in CODEC_H265): info.append('HEVC') return quality, info except: log_utils.error() return 'SD', []
def convert_size(size_bytes, to='GB'): try: import math if size_bytes == 0: return 0, '' power = { 'B': 0, 'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4, 'EB': 5, 'ZB': 6, 'YB': 7 } i = power[to] p = math.pow(1024, i) float_size = round(size_bytes / p, 2) # if to == 'B' or to == 'KB': # return 0, '' str_size = "%s %s" % (float_size, to) return float_size, str_size except: log_utils.error() return 0, ''
def yandex(url): try: cookie = client.request(url, output='cookie') r = client.request(url, cookie=cookie) r = re.sub(r'[^\x00-\x7F]+', ' ', r) sk = re.findall('"sk"\s*:\s*"([^"]+)', r)[0] idstring = re.findall('"id"\s*:\s*"([^"]+)', r)[0] idclient = binascii.b2a_hex(os.urandom(16)) post = { 'idClient': idclient, 'version': '3.9.2', 'sk': sk, '_model.0': 'do-get-resource-url', 'id.0': idstring } post = urlencode(post) r = client.request('https://yadi.sk/models/?_m=do-get-resource-url', post=post, cookie=cookie) r = json.loads(r) url = r['models'][0]['data']['file'] return url except: log_utils.error() return
def release_title_strip(release_title): try: try: release_title = release_title.encode('utf-8') except: pass release_title = release_title.lower().replace( "'", "").lstrip('.').rstrip('.') fmt = re.sub('[^a-z0-9]+', '.', release_title) fmt = '.%s.' % fmt fmt = re.sub( r'(.+)((?:19|20)[0-9]{2}|season.\d+|s[0-3]{1}[0-9]{1}|e\d+|complete)(.complete\.|.episode\.\d+\.|.episodes\.\d+\.\d+\.|.series|.extras|.ep\.\d+\.|.\d{1,2}\.|-|\.|\s)', '', fmt) # new for pack files # Fails for these cases # release_title = Game.of.Thrones.S01.1080p.BluRay.10bit.HEVC-MkvCage.Season.1.One (ENCODED) # fmt = .one. # .yelowstone.season.03.hamsterstudio.2019. # fmt = '' # may be best to pass "tvshowtitle" and "ep_title" and strip that off, as well as strip seaon/s01 type info off if fmt == '': return None else: return '.%s' % fmt except: log_utils.error() return None
def getTraktAsJson(url, post=None): try: r = __getTrakt(url, post) r = utils.json_loads_as_str(r) return r except: log_utils.error() pass
def getMovieTranslation(id, lang, full=False): try: url = '/movies/%s/translations/%s' % (id, lang) item = getTraktAsJson(url)[0] return item if full else item.get('title') except: log_utils.error() pass
def is_anime(content, type, type_id): from openscrapers.modules import trakt try: r = trakt.getGenre(content, type, type_id) return 'anime' in r or 'animation' in r except: log_utils.error() return False
def getGenre(content, type, type_id): try: r = '/search/%s/%s?type=%s&extended=full' % (type, type_id, content) r = getTraktAsJson(r) r = r[0].get(content, {}).get('genres', []) return r except: log_utils.error() return []
def strip_domain(url): try: if url.lower().startswith('http') or url.startswith('/'): url = re.findall('(?://.+?|)(/.+)', url)[0] url = client.replaceHTMLCodes(url) url = url.encode('utf-8') return url except: log_utils.error() return
def aliases_to_array(aliases, filter=None): try: if not filter: filter = [] if isinstance(filter, str): filter = [filter] return [x.get('title') for x in aliases if not filter or x.get('country') in filter] except: log_utils.error() return []
def get_size(url): # not called try: size = client.request(url, output='file_size') if size == '0': size = False float_size, str_size = convert_size(size) return float_size, str_size except: log_utils.error() return False
def getTVShowTranslation(id, lang, season=None, episode=None, full=False): try: if season and episode: url = '/shows/%s/seasons/%s/episodes/%s/translations/%s' % ( id, season, episode, lang) else: url = '/shows/%s/translations/%s' % (id, lang) item = getTraktAsJson(url)[0] return item if full else item.get('title') except: log_utils.error() pass
def cldmailru(url): try: v = url.split('public')[-1] r = client.request(url) r = re.sub(r'[^\x00-\x7F]+', ' ', r) # TODO Escaped dangling character {, can we confirm this needs to be here tok = re.findall('"tokens"\s*:\s*\{\s*"download"\s*:\s*"([^"]+)', r)[0] url = re.findall('"weblink_get"\s*:\s*\[.+?"url"\s*:\s*"([^"]+)', r)[0] url = '%s%s?key=%s' % (url, v, tok) return url except: log_utils.error() return
def _basic_request(url, headers=None, post=None, timeout='30', limit=None): try: try: headers.update(headers) except: headers = {} request = Request(url, data=post) _add_request_header(request, headers) response = urlopen(request, timeout=int(timeout)) return _get_result(response, limit) except: log_utils.error() return
def filter_single_episodes(hdlr, release_title): try: fmt = release_title_format(release_title) se = hdlr.lower() for item in [(se + r'(?:\.|-)e\d{1,2}(?:$|\.|-)'), (se + r'(?:\.|-)s[0-3]{1}[0-9]{1}e\d{1,2}(?:$|\.|-)'), (se + r'-\d{2}(?:$|\.|-)')]: if bool(re.search(item, fmt)): return False return True except: log_utils.error() return True
def check_url(url): try: url = url.lower() try: url = url.encode('utf-8') except: pass quality = get_qual(url) if not quality: quality = 'SD' return quality except: log_utils.error() return 'SD'
def release_title_format(release_title): try: release_title = release_title.lower().replace( "'", "").lstrip('.').rstrip('.') fmt = re.sub('[^a-z0-9-]+', '.', release_title) fmt = fmt.replace('.-.', '-').replace('-.', '-').replace('.-', '-').replace('--', '-') fmt = '.%s.' % fmt return fmt except: log_utils.error() return release_title
def label_to_quality(label): try: try: label = int(re.search('(\d+)', label).group(1)) except: label = 0 if label >= 2160: return '4K' elif 1920 <= label: return '1080p' elif 1280 <= label: return '720p' elif label <= 576: return 'SD' except: log_utils.error() return 'SD'
def check_title(title, name, hdlr, year): # from openscrapers.modules import log_utils try: match = True n = name.lower() h = hdlr.lower() t = n.split(h)[0].replace(year, '').replace('(', '').replace(')', '').replace('&', 'and').replace('.us.', '.') # log_utils.log('cleantitle.get(t) = %s' % cleantitle.get(t), log_utils.LOGDEBUG) # log_utils.log('cleantitle.get(title) = %s' % cleantitle.get(title), log_utils.LOGDEBUG) if cleantitle.get(t) != cleantitle.get(title): match = False if h not in n: match = False return match except: log_utils.error() match = False
def googlepass(url): try: try: headers = dict(parse_qsl(url.rsplit('|', 1)[1])) except: headers = None url = url.split('|')[0].replace('\\', '') url = client.request(url, headers=headers, output='geturl') if 'requiressl=yes' in url: url = url.replace('http://', 'https://') else: url = url.replace('https://', 'http://') if headers: url += '|%s' % urlencode(headers) return url except: log_utils.error() return
def clean_name(title, release_title): try: unwanted = [ '[zooqle.com]', '[horriblesubs]', '[.www.cpasbien.cm.]', '[.www.cpasbien.pw.]', '[auratorrent.pl].nastoletni.wilkoak', '[auratorrent.pl]', 'tamilrockers.com', 'www.tamilrockers.com', '[.oxtorrent.com.]', '[.www.torrenting.com.]', '[.Www.nextorrent.site.]', '[.oxtorrent.com.]', '[gktorrent.com]', 'www.torrenting.com', 'www.torrenting.org', 'www.torrent9.nz', '[.www.omgtorrent.com.]', '[.www.torrent9.uno.]', '[agusiq.torrents.pl]', '[katmoviehd.to]', '[3d.hentai]', '[dark.media]', '[filetracker.pl]', 'www-torrenting-com', 'www-torrenting-org', '[katmoviehd.eu]', 'www.scenetime.com', 'www.tamilrockerrs.pl', '[.torrent9.tv.]', '[nextorrent.net]', '+katmoviehd.pw+', 'www.movcr.tv', 'www.bludv.tv', '[www.torrent9.ph.]', '[acesse.]', '[acesse-hd-elite-me]', '[torrentcouch.net]', 'ramin.djawadi', '[prof]', '[reup]', '[ah]', '[ul]', '+13.+', 'taht.oyunlar', '[agusiq-torrents.pl]', 'agusiq-torrents-pl', 'crazy4tv.com', '[tv]' ] unwanted2 = [ '.', '..', '...', '{', '}', '[.]', '[.]', '[.', '+-+-', '-', '-.', '.-.' ] if release_title.lower().startswith('rifftrax'): return release_title release_title = strip_non_ascii_and_unprintable(release_title) release_title = release_title.lstrip('/ ') release_title = release_title.lower() release_title = release_title.replace(' ', '.') for i in unwanted: if release_title.startswith(i): release_title = release_title.replace(i, '') break for i in unwanted2: release_title = release_title.lstrip(i) # log_utils.log('final release_title: ' + str(release_title), log_utils.LOGDEBUG) return release_title except: log_utils.error()
def is_host_valid(url, domains): try: if any(x in url.lower() for x in ['.rar.', '.zip.', '.iso.']) or any( url.lower().endswith(x) for x in ['.rar', '.zip', '.iso']): return False, '' host = __top_domain(url) hosts = [domain.lower() for domain in domains if host and host in domain.lower()] if hosts and '.' not in host: host = hosts[0] if hosts and any([h for h in ['google', 'picasa', 'blogspot'] if h in host]): host = 'gvideo' if hosts and any([h for h in ['akamaized', 'ocloud'] if h in host]): host = 'CDN' return any(hosts), host except: log_utils.error() return False, ''
def check_for_addon_update(): log_utils.log('OpenScrapers checking available updates', log_utils.LOGNOTICE) try: import re import requests repo_xml = requests.get('https://raw.githubusercontent.com/a4k-openproject/repository.openscrapers/master/zips/addons.xml') if not repo_xml.status_code == 200: log_utils.log('Could not connect to repo XML, status: %s' % repo_xml.status_code, log_utils.LOGNOTICE) return repo_version = re.findall(r'<addon id=\"script.module.openscrapers\".*version=\"(\d*.\d*.\d*.\d*)\"', repo_xml.text)[0] local_version = control.addonVersion() if control.check_version_numbers(local_version, repo_version): while control.condVisibility('Library.IsScanningVideo'): control.sleep(10000) log_utils.log('A newer version of OpenScrapers is available. Installed Version: v%s, Repo Version: v%s' % (local_version, repo_version), log_utils.LOGNOTICE) control.notification(title = 'default', message = 'A new verison of OpenScrapers is available from the repository. Please consider updating to v%s' % repo_version, icon = 'default', time=5000, sound=False) except: log_utils.error() pass
def vk(url): try: query = parse_qs(urlparse(url).query) try: oid, video_id = query['oid'][0], query['id'][0] except: oid, video_id = re.findall('\/video(.*)_(.*)', url)[0] sources_url = 'http://vk.com/al_video.php?act=show_inline&al=1&video=%s_%s' % ( oid, video_id) html = client.request(sources_url) html = re.sub(r'[^\x00-\x7F]+', ' ', html) sources = re.findall('(\d+)x\d+.+?(http.+?\.m3u8.+?)n', html) if not sources: sources = re.findall('"url(\d+)"\s*:\s*"(.+?)"', html) sources = [(i[0], i[1].replace('\\', '')) for i in sources] sources = dict(sources) url = [] try: url += [{'quality': '720p', 'url': sources['720']}] except: pass try: url += [{'quality': 'SD', 'url': sources['540']}] except: pass try: url += [{'quality': 'SD', 'url': sources['480']}] except: pass if not url == []: return url try: url += [{'quality': 'SD', 'url': sources['360']}] except: pass if not url == []: return url try: url += [{'quality': 'SD', 'url': sources['240']}] except: pass if not url == []: return url except: log_utils.error() return
def url_strip(url): try: url = unquote_plus(url) if 'magnet' in url: url = url.split('&dn=')[1] url = url.lower().replace("'", "").lstrip('.').rstrip('.') fmt = re.sub('[^a-z0-9]+', '.', url) fmt = '.%s.' % fmt fmt = re.sub( r'(.+)((?:19|20)[0-9]{2}|season.\d+|s[0-3]{1}[0-9]{1}|e\d+|complete)(.complete\.|.episode\.\d+\.|.episodes\.\d+\.\d+\.|.series|.extras|.ep\.\d+\.|.\d{1,2}\.|-|\.|\s)', '', fmt) # new for pack files if '.http' in fmt: fmt = None if fmt == '': return None else: return '.%s' % fmt except: log_utils.error() return None
def cache_get(key): try: cursor = _get_connection_cursor() cursor.execute( "SELECT * FROM sqlite_master WHERE type='table' AND name='%s';" % cache_table) ck_table = cursor.fetchone() if not ck_table: cursor.close() return None cursor.execute("SELECT * FROM %s WHERE key = ?" % cache_table, [key]) results = cursor.fetchone() cursor.close() return results except: log_utils.error() try: cursor.close() except: pass return None
def cache_insert(key, value): try: cursor = _get_connection_cursor() now = int(time.time()) cursor.execute( "CREATE TABLE IF NOT EXISTS %s (key TEXT, value TEXT, date INTEGER, UNIQUE(key))" % cache_table) update_result = cursor.execute( "UPDATE %s SET value=?,date=? WHERE key=?" % cache_table, (value, now, key)) if update_result.rowcount is 0: cursor.execute("INSERT INTO %s Values (?, ?, ?)" % cache_table, (key, value, now)) cursor.connection.commit() cursor.close() except: log_utils.error() try: cursor.close() except: pass
def remove_lang(release_title, episode_title=None): try: fmt = release_title_strip(release_title) if fmt is None: return False # log_utils.log('fmt = %s for release_title = %s' % (str(fmt), str(release_title)), __name__, log_utils.LOGDEBUG) if episode_title: episode_title = episode_title.lower().replace("'", "") episode_title = re.sub('[^a-z0-9]+', '.', episode_title) fmt = '%s' % re.sub(episode_title, '', fmt) filter_undesirables = control.setting('filter.undesirables') == 'true' filter_foreign_single_audio = control.setting( 'filter.foreign.single.audio') == 'true' if filter_undesirables: if any(value in fmt for value in UNDESIREABLES): return True if any(value in fmt for value in DUBBED): return True if any(value in fmt for value in SUBS): return True # if any(value in fmt for value in ADDS): # return True if filter_foreign_single_audio: if any(value in fmt for value in LANG) and not any( value in fmt for value in ['.eng.', '.en.', 'english']): return True if any(value in fmt for value in ABV_LANG) and not any( value in fmt for value in ['.eng.', '.en.', 'english']): return True if fmt.endswith('.srt.') and not any( value in fmt for value in ['with.srt', '.avi', '.mkv', '.mp4']): return True return False except: log_utils.error() return False
def check_title(title, aliases, release_title, hdlr, year): try: aliases = json.loads(aliases) aliases = aliases_to_array(aliases) except: aliases = None # log_utils.log('aliases = %s' % str(aliases), __name__, log_utils.LOGDEBUG) # log_utils.log('aliases type = %s' % type(aliases), __name__, log_utils.LOGDEBUG) title_list = [] if aliases: for item in aliases: alias = item.replace('!', '').replace('(', '').replace(')', '') # alias = re.sub(r'[^A-Za-z0-9\s\.-]+', '', item) for match in re.finditer(r'(.+?)(U\.S|U\.K)', alias): alias = match.group(1) + match.group(2).replace('.', '') title_list.append(alias) try: release_title = release_title_format(release_title) match = True title = title.replace('!', '').replace('(', '').replace(')', '') # title = re.sub(r'[^A-Za-z0-9\s\.-]+', '', title) title_list.append(title) n = release_title h = hdlr.lower() t = n.split(h)[0].replace(year, '').replace('(', '').replace( ')', '').replace('&', 'and') if all(cleantitle.get(i) != cleantitle.get(t) for i in title_list): match = False if h not in n: match = False return match except: log_utils.error() match = False
def get_release_quality(release_name, release_link=None): if release_name is None: return try: release_name = release_name.encode('utf-8') except: pass try: quality = None release_name = release_name.upper() fmt = re.sub('(.+)(\d{4}|S\d+E\d+)(\.|\)\.|\)|\]\.|\]|\s)', '', release_name) # log_utils.log('fmt = %s' % fmt, log_utils.LOGDEBUG) fmt = fmt.lower() quality = get_qual(fmt) if not quality: if release_link: release_link = release_link.lower() try: release_link = release_link.encode('utf-8') except: pass quality = get_qual(release_link) if not quality: quality = 'SD' else: quality = 'SD' info = [] if any(value in fmt for value in VIDEO_3D): info.append('3D') if any(value in fmt for value in CODEC_H265): info.append('HEVC') return quality, info except: log_utils.error() return 'SD', []