def get_release_quality(release_title, release_link=None):
try:
quality = None
fmt = release_title_strip(release_title)
if fmt is not None:
quality = get_qual(fmt)
if not quality:
if release_link:
release_link = release_link.lower()
try:
release_link = release_link.encode('utf-8')
except:
pass
quality = get_qual(release_link)
if not quality:
quality = 'SD'
else:
quality = 'SD'
info = []
if fmt is not None:
if any(value in fmt for value in VIDEO_3D):
info.append('3D')
if any(value in fmt for value in CODEC_H265):
info.append('HEVC')
return quality, info
except:
log_utils.error()
return 'SD', []
def convert_size(size_bytes, to='GB'):
try:
import math
if size_bytes == 0:
return 0, ''
power = {
'B': 0,
'KB': 1,
'MB': 2,
'GB': 3,
'TB': 4,
'EB': 5,
'ZB': 6,
'YB': 7
}
i = power[to]
p = math.pow(1024, i)
float_size = round(size_bytes / p, 2)
# if to == 'B' or to == 'KB':
# return 0, ''
str_size = "%s %s" % (float_size, to)
return float_size, str_size
except:
log_utils.error()
return 0, ''
def yandex(url):
try:
cookie = client.request(url, output='cookie')
r = client.request(url, cookie=cookie)
r = re.sub(r'[^\x00-\x7F]+', ' ', r)
sk = re.findall('"sk"\s*:\s*"([^"]+)', r)[0]
idstring = re.findall('"id"\s*:\s*"([^"]+)', r)[0]
idclient = binascii.b2a_hex(os.urandom(16))
post = {
'idClient': idclient,
'version': '3.9.2',
'sk': sk,
'_model.0': 'do-get-resource-url',
'id.0': idstring
}
post = urlencode(post)
r = client.request('https://yadi.sk/models/?_m=do-get-resource-url',
post=post,
cookie=cookie)
r = json.loads(r)
url = r['models'][0]['data']['file']
return url
except:
log_utils.error()
return
def release_title_strip(release_title):
try:
try:
release_title = release_title.encode('utf-8')
except:
pass
release_title = release_title.lower().replace(
"'", "").lstrip('.').rstrip('.')
fmt = re.sub('[^a-z0-9]+', '.', release_title)
fmt = '.%s.' % fmt
fmt = re.sub(
r'(.+)((?:19|20)[0-9]{2}|season.\d+|s[0-3]{1}[0-9]{1}|e\d+|complete)(.complete\.|.episode\.\d+\.|.episodes\.\d+\.\d+\.|.series|.extras|.ep\.\d+\.|.\d{1,2}\.|-|\.|\s)',
'', fmt) # new for pack files
# Fails for these cases
# release_title = Game.of.Thrones.S01.1080p.BluRay.10bit.HEVC-MkvCage.Season.1.One (ENCODED)
# fmt = .one.
# .yelowstone.season.03.hamsterstudio.2019.
# fmt = ''
# may be best to pass "tvshowtitle" and "ep_title" and strip that off, as well as strip seaon/s01 type info off
if fmt == '':
return None
else:
return '.%s' % fmt
except:
log_utils.error()
return None
def getTraktAsJson(url, post=None):
try:
r = __getTrakt(url, post)
r = utils.json_loads_as_str(r)
return r
except:
log_utils.error()
pass
def getMovieTranslation(id, lang, full=False):
try:
url = '/movies/%s/translations/%s' % (id, lang)
item = getTraktAsJson(url)[0]
return item if full else item.get('title')
except:
log_utils.error()
pass
def is_anime(content, type, type_id):
from openscrapers.modules import trakt
try:
r = trakt.getGenre(content, type, type_id)
return 'anime' in r or 'animation' in r
except:
log_utils.error()
return False
def getGenre(content, type, type_id):
try:
r = '/search/%s/%s?type=%s&extended=full' % (type, type_id, content)
r = getTraktAsJson(r)
r = r[0].get(content, {}).get('genres', [])
return r
except:
log_utils.error()
return []
def strip_domain(url):
try:
if url.lower().startswith('http') or url.startswith('/'):
url = re.findall('(?://.+?|)(/.+)', url)[0]
url = client.replaceHTMLCodes(url)
url = url.encode('utf-8')
return url
except:
log_utils.error()
return
def aliases_to_array(aliases, filter=None):
try:
if not filter:
filter = []
if isinstance(filter, str):
filter = [filter]
return [x.get('title') for x in aliases if not filter or x.get('country') in filter]
except:
log_utils.error()
return []
def get_size(url): # not called try: size = client.request(url, output='file_size') if size == '0': size = False float_size, str_size = convert_size(size) return float_size, str_size except: log_utils.error() return False
def getTVShowTranslation(id, lang, season=None, episode=None, full=False):
try:
if season and episode:
url = '/shows/%s/seasons/%s/episodes/%s/translations/%s' % (
id, season, episode, lang)
else:
url = '/shows/%s/translations/%s' % (id, lang)
item = getTraktAsJson(url)[0]
return item if full else item.get('title')
except:
log_utils.error()
pass
def cldmailru(url):
try:
v = url.split('public')[-1]
r = client.request(url)
r = re.sub(r'[^\x00-\x7F]+', ' ', r)
# TODO Escaped dangling character {, can we confirm this needs to be here
tok = re.findall('"tokens"\s*:\s*\{\s*"download"\s*:\s*"([^"]+)', r)[0]
url = re.findall('"weblink_get"\s*:\s*\[.+?"url"\s*:\s*"([^"]+)', r)[0]
url = '%s%s?key=%s' % (url, v, tok)
return url
except:
log_utils.error()
return
def _basic_request(url, headers=None, post=None, timeout='30', limit=None):
try:
try:
headers.update(headers)
except:
headers = {}
request = Request(url, data=post)
_add_request_header(request, headers)
response = urlopen(request, timeout=int(timeout))
return _get_result(response, limit)
except:
log_utils.error()
return
def filter_single_episodes(hdlr, release_title):
try:
fmt = release_title_format(release_title)
se = hdlr.lower()
for item in [(se + r'(?:\.|-)e\d{1,2}(?:$|\.|-)'),
(se + r'(?:\.|-)s[0-3]{1}[0-9]{1}e\d{1,2}(?:$|\.|-)'),
(se + r'-\d{2}(?:$|\.|-)')]:
if bool(re.search(item, fmt)):
return False
return True
except:
log_utils.error()
return True
def check_url(url):
try:
url = url.lower()
try:
url = url.encode('utf-8')
except:
pass
quality = get_qual(url)
if not quality:
quality = 'SD'
return quality
except:
log_utils.error()
return 'SD'
def release_title_format(release_title):
try:
release_title = release_title.lower().replace(
"'", "").lstrip('.').rstrip('.')
fmt = re.sub('[^a-z0-9-]+', '.', release_title)
fmt = fmt.replace('.-.',
'-').replace('-.',
'-').replace('.-',
'-').replace('--', '-')
fmt = '.%s.' % fmt
return fmt
except:
log_utils.error()
return release_title
def label_to_quality(label):
try:
try:
label = int(re.search('(\d+)', label).group(1))
except:
label = 0
if label >= 2160:
return '4K'
elif 1920 <= label:
return '1080p'
elif 1280 <= label:
return '720p'
elif label <= 576:
return 'SD'
except:
log_utils.error()
return 'SD'
def check_title(title, name, hdlr, year):
# from openscrapers.modules import log_utils
try:
match = True
n = name.lower()
h = hdlr.lower()
t = n.split(h)[0].replace(year, '').replace('(', '').replace(')', '').replace('&', 'and').replace('.us.', '.')
# log_utils.log('cleantitle.get(t) = %s' % cleantitle.get(t), log_utils.LOGDEBUG)
# log_utils.log('cleantitle.get(title) = %s' % cleantitle.get(title), log_utils.LOGDEBUG)
if cleantitle.get(t) != cleantitle.get(title):
match = False
if h not in n:
match = False
return match
except:
log_utils.error()
match = False
def googlepass(url):
try:
try:
headers = dict(parse_qsl(url.rsplit('|', 1)[1]))
except:
headers = None
url = url.split('|')[0].replace('\\', '')
url = client.request(url, headers=headers, output='geturl')
if 'requiressl=yes' in url:
url = url.replace('http://', 'https://')
else:
url = url.replace('https://', 'http://')
if headers: url += '|%s' % urlencode(headers)
return url
except:
log_utils.error()
return
def clean_name(title, release_title):
try:
unwanted = [
'[zooqle.com]', '[horriblesubs]', '[.www.cpasbien.cm.]',
'[.www.cpasbien.pw.]', '[auratorrent.pl].nastoletni.wilkoak',
'[auratorrent.pl]', 'tamilrockers.com', 'www.tamilrockers.com',
'[.oxtorrent.com.]', '[.www.torrenting.com.]',
'[.Www.nextorrent.site.]', '[.oxtorrent.com.]', '[gktorrent.com]',
'www.torrenting.com', 'www.torrenting.org', 'www.torrent9.nz',
'[.www.omgtorrent.com.]', '[.www.torrent9.uno.]',
'[agusiq.torrents.pl]', '[katmoviehd.to]', '[3d.hentai]',
'[dark.media]', '[filetracker.pl]', 'www-torrenting-com',
'www-torrenting-org', '[katmoviehd.eu]', 'www.scenetime.com',
'www.tamilrockerrs.pl', '[.torrent9.tv.]', '[nextorrent.net]',
'+katmoviehd.pw+', 'www.movcr.tv', 'www.bludv.tv',
'[www.torrent9.ph.]', '[acesse.]', '[acesse-hd-elite-me]',
'[torrentcouch.net]', 'ramin.djawadi', '[prof]', '[reup]', '[ah]',
'[ul]', '+13.+', 'taht.oyunlar', '[agusiq-torrents.pl]',
'agusiq-torrents-pl', 'crazy4tv.com', '[tv]'
]
unwanted2 = [
'.', '..', '...', '{', '}', '[.]', '[.]', '[.', '+-+-', '-', '-.',
'.-.'
]
if release_title.lower().startswith('rifftrax'):
return release_title
release_title = strip_non_ascii_and_unprintable(release_title)
release_title = release_title.lstrip('/ ')
release_title = release_title.lower()
release_title = release_title.replace(' ', '.')
for i in unwanted:
if release_title.startswith(i):
release_title = release_title.replace(i, '')
break
for i in unwanted2:
release_title = release_title.lstrip(i)
# log_utils.log('final release_title: ' + str(release_title), log_utils.LOGDEBUG)
return release_title
except:
log_utils.error()
def is_host_valid(url, domains): try: if any(x in url.lower() for x in ['.rar.', '.zip.', '.iso.']) or any( url.lower().endswith(x) for x in ['.rar', '.zip', '.iso']): return False, '' host = __top_domain(url) hosts = [domain.lower() for domain in domains if host and host in domain.lower()] if hosts and '.' not in host: host = hosts[0] if hosts and any([h for h in ['google', 'picasa', 'blogspot'] if h in host]): host = 'gvideo' if hosts and any([h for h in ['akamaized', 'ocloud'] if h in host]): host = 'CDN' return any(hosts), host except: log_utils.error() return False, ''
def check_for_addon_update():
log_utils.log('OpenScrapers checking available updates', log_utils.LOGNOTICE)
try:
import re
import requests
repo_xml = requests.get('https://raw.githubusercontent.com/a4k-openproject/repository.openscrapers/master/zips/addons.xml')
if not repo_xml.status_code == 200:
log_utils.log('Could not connect to repo XML, status: %s' % repo_xml.status_code, log_utils.LOGNOTICE)
return
repo_version = re.findall(r'<addon id=\"script.module.openscrapers\".*version=\"(\d*.\d*.\d*.\d*)\"', repo_xml.text)[0]
local_version = control.addonVersion()
if control.check_version_numbers(local_version, repo_version):
while control.condVisibility('Library.IsScanningVideo'):
control.sleep(10000)
log_utils.log('A newer version of OpenScrapers is available. Installed Version: v%s, Repo Version: v%s' % (local_version, repo_version), log_utils.LOGNOTICE)
control.notification(title = 'default', message = 'A new verison of OpenScrapers is available from the repository. Please consider updating to v%s' % repo_version, icon = 'default', time=5000, sound=False)
except:
log_utils.error()
pass
def vk(url):
try:
query = parse_qs(urlparse(url).query)
try:
oid, video_id = query['oid'][0], query['id'][0]
except:
oid, video_id = re.findall('\/video(.*)_(.*)', url)[0]
sources_url = 'http://vk.com/al_video.php?act=show_inline&al=1&video=%s_%s' % (
oid, video_id)
html = client.request(sources_url)
html = re.sub(r'[^\x00-\x7F]+', ' ', html)
sources = re.findall('(\d+)x\d+.+?(http.+?\.m3u8.+?)n', html)
if not sources:
sources = re.findall('"url(\d+)"\s*:\s*"(.+?)"', html)
sources = [(i[0], i[1].replace('\\', '')) for i in sources]
sources = dict(sources)
url = []
try:
url += [{'quality': '720p', 'url': sources['720']}]
except:
pass
try:
url += [{'quality': 'SD', 'url': sources['540']}]
except:
pass
try:
url += [{'quality': 'SD', 'url': sources['480']}]
except:
pass
if not url == []: return url
try:
url += [{'quality': 'SD', 'url': sources['360']}]
except:
pass
if not url == []: return url
try:
url += [{'quality': 'SD', 'url': sources['240']}]
except:
pass
if not url == []: return url
except:
log_utils.error()
return
def url_strip(url):
try:
url = unquote_plus(url)
if 'magnet' in url:
url = url.split('&dn=')[1]
url = url.lower().replace("'", "").lstrip('.').rstrip('.')
fmt = re.sub('[^a-z0-9]+', '.', url)
fmt = '.%s.' % fmt
fmt = re.sub(
r'(.+)((?:19|20)[0-9]{2}|season.\d+|s[0-3]{1}[0-9]{1}|e\d+|complete)(.complete\.|.episode\.\d+\.|.episodes\.\d+\.\d+\.|.series|.extras|.ep\.\d+\.|.\d{1,2}\.|-|\.|\s)',
'', fmt) # new for pack files
if '.http' in fmt:
fmt = None
if fmt == '':
return None
else:
return '.%s' % fmt
except:
log_utils.error()
return None
def cache_get(key):
try:
cursor = _get_connection_cursor()
cursor.execute(
"SELECT * FROM sqlite_master WHERE type='table' AND name='%s';" %
cache_table)
ck_table = cursor.fetchone()
if not ck_table:
cursor.close()
return None
cursor.execute("SELECT * FROM %s WHERE key = ?" % cache_table, [key])
results = cursor.fetchone()
cursor.close()
return results
except:
log_utils.error()
try:
cursor.close()
except:
pass
return None
def cache_insert(key, value):
try:
cursor = _get_connection_cursor()
now = int(time.time())
cursor.execute(
"CREATE TABLE IF NOT EXISTS %s (key TEXT, value TEXT, date INTEGER, UNIQUE(key))"
% cache_table)
update_result = cursor.execute(
"UPDATE %s SET value=?,date=? WHERE key=?" % cache_table,
(value, now, key))
if update_result.rowcount is 0:
cursor.execute("INSERT INTO %s Values (?, ?, ?)" % cache_table,
(key, value, now))
cursor.connection.commit()
cursor.close()
except:
log_utils.error()
try:
cursor.close()
except:
pass
def remove_lang(release_title, episode_title=None):
try:
fmt = release_title_strip(release_title)
if fmt is None:
return False
# log_utils.log('fmt = %s for release_title = %s' % (str(fmt), str(release_title)), __name__, log_utils.LOGDEBUG)
if episode_title:
episode_title = episode_title.lower().replace("'", "")
episode_title = re.sub('[^a-z0-9]+', '.', episode_title)
fmt = '%s' % re.sub(episode_title, '', fmt)
filter_undesirables = control.setting('filter.undesirables') == 'true'
filter_foreign_single_audio = control.setting(
'filter.foreign.single.audio') == 'true'
if filter_undesirables:
if any(value in fmt for value in UNDESIREABLES):
return True
if any(value in fmt for value in DUBBED):
return True
if any(value in fmt for value in SUBS):
return True
# if any(value in fmt for value in ADDS):
# return True
if filter_foreign_single_audio:
if any(value in fmt for value in LANG) and not any(
value in fmt for value in ['.eng.', '.en.', 'english']):
return True
if any(value in fmt for value in ABV_LANG) and not any(
value in fmt for value in ['.eng.', '.en.', 'english']):
return True
if fmt.endswith('.srt.') and not any(
value in fmt
for value in ['with.srt', '.avi', '.mkv', '.mp4']):
return True
return False
except:
log_utils.error()
return False
def check_title(title, aliases, release_title, hdlr, year):
try:
aliases = json.loads(aliases)
aliases = aliases_to_array(aliases)
except:
aliases = None
# log_utils.log('aliases = %s' % str(aliases), __name__, log_utils.LOGDEBUG)
# log_utils.log('aliases type = %s' % type(aliases), __name__, log_utils.LOGDEBUG)
title_list = []
if aliases:
for item in aliases:
alias = item.replace('!', '').replace('(', '').replace(')', '')
# alias = re.sub(r'[^A-Za-z0-9\s\.-]+', '', item)
for match in re.finditer(r'(.+?)(U\.S|U\.K)', alias):
alias = match.group(1) + match.group(2).replace('.', '')
title_list.append(alias)
try:
release_title = release_title_format(release_title)
match = True
title = title.replace('!', '').replace('(', '').replace(')', '')
# title = re.sub(r'[^A-Za-z0-9\s\.-]+', '', title)
title_list.append(title)
n = release_title
h = hdlr.lower()
t = n.split(h)[0].replace(year, '').replace('(', '').replace(
')', '').replace('&', 'and')
if all(cleantitle.get(i) != cleantitle.get(t) for i in title_list):
match = False
if h not in n:
match = False
return match
except:
log_utils.error()
match = False
def get_release_quality(release_name, release_link=None):
if release_name is None:
return
try:
release_name = release_name.encode('utf-8')
except:
pass
try:
quality = None
release_name = release_name.upper()
fmt = re.sub('(.+)(\d{4}|S\d+E\d+)(\.|\)\.|\)|\]\.|\]|\s)', '', release_name)
# log_utils.log('fmt = %s' % fmt, log_utils.LOGDEBUG)
fmt = fmt.lower()
quality = get_qual(fmt)
if not quality:
if release_link:
release_link = release_link.lower()
try:
release_link = release_link.encode('utf-8')
except:
pass
quality = get_qual(release_link)
if not quality:
quality = 'SD'
else:
quality = 'SD'
info = []
if any(value in fmt for value in VIDEO_3D):
info.append('3D')
if any(value in fmt for value in CODEC_H265):
info.append('HEVC')
return quality, info
except:
log_utils.error()
return 'SD', []
