def createBaseUrl(self):
host = Env.setting('host')
if host == '0.0.0.0' or host == '':
host = 'localhost'
port = Env.setting('port')
ssl = Env.setting('ssl_cert') and Env.setting('ssl_key')
return '%s:%d%s' % (cleanHost(host, ssl = ssl).rstrip('/'), int(port), Env.get('web_base'))
def get_current_user(self):
username = Env.setting('username')
password = Env.setting('password')
if username and password:
return self.get_secure_cookie('user')
else: # Login when no username or password are set
return True
def calculate(self, nzb, movie):
""" Calculate the score of a NZB, used for sorting later """
# Merge global and category
preferred_words = splitString(Env.setting('preferred_words', section = 'searcher').lower())
try: preferred_words = removeDuplicate(preferred_words + splitString(movie['category']['preferred'].lower()))
except: pass
score = nameScore(toUnicode(nzb['name']), movie['info']['year'], preferred_words)
for movie_title in movie['info']['titles']:
score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title))
score += namePositionScore(toUnicode(nzb['name']), toUnicode(movie_title))
score += sizeScore(nzb['size'])
# Torrents only
if nzb.get('seeders'):
try:
score += nzb.get('seeders') * 100 / 15
score += nzb.get('leechers') * 100 / 30
except:
pass
# Provider score
score += providerScore(nzb['provider'])
# Duplicates in name
score += duplicateScore(nzb['name'], getTitle(movie))
# Merge global and category
ignored_words = splitString(Env.setting('ignored_words', section = 'searcher').lower())
try: ignored_words = removeDuplicate(ignored_words + splitString(movie['category']['ignored'].lower()))
except: pass
# Partial ignored words
score += partialIgnoredScore(nzb['name'], getTitle(movie), ignored_words)
# Ignore single downloads from multipart
score += halfMultipartScore(nzb['name'])
# Extra provider specific check
extra_score = nzb.get('extra_score')
if extra_score:
score += extra_score(nzb)
# Scene / Nuke scoring
score += sceneScore(nzb['name'])
return score
def post(self, *args, **kwargs):
api_key = None
username = Env.setting('username')
password = Env.setting('password')
if (self.get_argument('username') == username or not username) and (md5(self.get_argument('password')) == password or not password):
api_key = Env.setting('api_key')
if api_key:
remember_me = tryInt(self.get_argument('remember_me', default = 0))
self.set_secure_cookie('user', api_key, expires_days = 30 if remember_me > 0 else None)
self.redirect(Env.get('web_base'))
def getRequestHeaders(self):
return {
'X-CP-Version': fireEvent('app.version', single = True),
'X-CP-API': self.api_version,
'X-CP-Time': time.time(),
'X-CP-Identifier': '+%s' % Env.setting('api_key', 'core')[:10], # Use first 10 as identifier, so we don't need to use IP address in api stats
}
def get(self, *args, **kwargs):
api_key = None
try:
username = Env.setting('username')
password = Env.setting('password')
if (self.get_argument('u') == md5(username) or not username) and (self.get_argument('p') == password or not password):
api_key = Env.setting('api_key')
self.write({
'success': api_key is not None,
'api_key': api_key
})
except:
log.error('Failed doing key request: %s', (traceback.format_exc()))
self.write({'success': False, 'error': 'Failed returning results'})
def providerScore(provider):
try:
score = tryInt(Env.setting('extra_score', section = provider.lower(), default = 0))
except:
score = 0
return score
def fillResult(self, result):
defaults = {
'id': 0,
'protocol': self.provider.protocol,
'type': self.provider.type,
'provider': self.provider.getName(),
'download': self.provider.loginDownload if self.provider.urls.get('login') else self.provider.download,
'seed_ratio': Env.setting('seed_ratio', section = self.provider.getName().lower(), default = ''),
'seed_time': Env.setting('seed_time', section = self.provider.getName().lower(), default = ''),
'url': '',
'name': '',
'age': 0,
'size': 0,
'description': '',
'score': 0
}
return mergeDicts(defaults, result)
def launchBrowser(self):
if Env.setting('launch_browser'):
log.info('Launching browser')
url = self.createBaseUrl()
try:
webbrowser.open(url, 2, 1)
except:
try:
webbrowser.open(url, 1, 1)
except:
log.error('Could not launch a browser.')
def tryDownloadResult(self, results, media, quality_custom):
wait_for = False
let_through = False
filtered_results = []
minimum_seeders = tryInt(Env.setting('minimum_seeders', section = 'torrent', default = 1))
# Filter out ignored and other releases we don't want
for rel in results:
if rel['status'] in ['ignored', 'failed']:
log.info('Ignored: %s', rel['name'])
continue
if rel['score'] < quality_custom.get('minimum_score'):
log.info('Ignored, score "%s" to low, need at least "%s": %s', (rel['score'], quality_custom.get('minimum_score'), rel['name']))
continue
if rel['size'] <= 50:
log.info('Ignored, size "%sMB" to low: %s', (rel['size'], rel['name']))
continue
if 'seeders' in rel and rel.get('seeders') < minimum_seeders:
log.info('Ignored, not enough seeders, has %s needs %s: %s', (rel.get('seeders'), minimum_seeders, rel['name']))
continue
# If a single release comes through the "wait for", let through all
rel['wait_for'] = False
if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0):
rel['wait_for'] = True
else:
let_through = True
filtered_results.append(rel)
# Loop through filtered results
for rel in filtered_results:
# Only wait if not a single release is old enough
if rel.get('wait_for') and not let_through:
log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for') - rel.get('age'), rel['name']))
wait_for = True
continue
downloaded = fireEvent('release.download', data = rel, media = media, single = True)
if downloaded is True:
return True
elif downloaded != 'try_next':
break
return wait_for
def cpTag(self, media, unique_tag = False):
tag = ''
if Env.setting('enabled', 'renamer') or unique_tag:
identifier = getIdentifier(media) or ''
unique_tag = ', ' + randomString() if unique_tag else ''
tag = '.cp('
tag += identifier
tag += ', ' if unique_tag and identifier else ''
tag += randomString() if unique_tag else ''
tag += ')'
return tag if len(tag) > 7 else ''
def buildUrl(self, media, quality):
query = tryUrlencode(
{
"q": getIdentifier(media),
"m": "n",
"max": 400,
"adv_age": Env.setting("retention", "nzb"),
"adv_sort": "date",
"adv_col": "on",
"adv_nfo": "on",
"minsize": quality.get("size_min"),
"maxsize": quality.get("size_max"),
}
)
return query
def __init__(self):
desktop = Env.get('desktop')
desktop.setSettings({
'base_url': fireEvent('app.base_url', single = True),
'api_url': fireEvent('app.api_url', single = True),
'api': Env.setting('api'),
})
# Events from desktop
desktop.addEvents({
'onClose': self.onClose,
})
# Events to desktop
addEvent('app.after_shutdown', desktop.afterShutdown)
addEvent('app.load', desktop.onAppLoad, priority = 110)
def get(self, random, route):
bookmarklet_host = Env.setting('bookmarklet_host')
loc = bookmarklet_host if bookmarklet_host else "{0}://{1}".format(self.request.protocol, self.request.headers.get('X-Forwarded-Host') or self.request.headers.get('host'))
params = {
'includes': fireEvent('userscript.get_includes', merge = True),
'excludes': fireEvent('userscript.get_excludes', merge = True),
'version': klass.getVersion(),
'api': '%suserscript/' % Env.get('api_base'),
'host': loc,
}
script = klass.renderTemplate(__file__, 'template.js_tmpl', **params)
klass.createFile(os.path.join(Env.get('cache_dir'), 'whatpotato.user.js'), script)
self.redirect(Env.get('api_base') + 'file.cache/whatpotato.user.js')
def manifest(handler):
web_base = Env.get('web_base')
static_base = Env.get('static_path')
lines = [
'CACHE MANIFEST',
'# %s theme' % ('dark' if Env.setting('dark_theme') else 'light'),
'',
'CACHE:',
''
]
if not Env.get('dev'):
# CSS
for url in fireEvent('clientscript.get_styles', single = True):
lines.append(web_base + url)
# Scripts
for url in fireEvent('clientscript.get_scripts', single = True):
lines.append(web_base + url)
# Favicon
lines.append(static_base + 'images/favicon.ico')
# Fonts
font_folder = sp(os.path.join(Env.get('app_dir'), 'whatpotato', 'static', 'fonts'))
for subfolder, dirs, files in os.walk(font_folder, topdown = False):
for file in files:
if '.woff' in file:
lines.append(static_base + 'fonts/' + file + ('?%s' % os.path.getmtime(os.path.join(font_folder, file))))
else:
lines.append('# Not caching anything in dev mode')
# End lines
lines.extend(['',
'NETWORK: ',
'*'])
handler.set_header('Content-Type', 'text/cache-manifest')
return '\n'.join(lines)
def __init__(self):
# Get options via arg
from whatpotato.runner import getOptions
self.options = getOptions(sys.argv[1:])
# Load settings
settings = Env.get('settings')
settings.setFile(self.options.config_file)
# Create data dir if needed
if self.options.data_dir:
self.data_dir = self.options.data_dir
else:
self.data_dir = os.path.expanduser(Env.setting('data_dir'))
if self.data_dir == '':
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, 'logs');
if not os.path.isdir(self.log_dir):
os.makedirs(self.log_dir)
# Logging
from whatpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def getMinimal(self, min_type):
return Env.setting(min_type, 'automation')
def createApiUrl(self):
return '%sapi/%s' % (self.createBaseUrl(), Env.setting('api_key'))
def download(self, data, media, manual = False):
# Test to see if any downloaders are enabled for this type
downloader_enabled = fireEvent('download.enabled', manual, data, single = True)
if not downloader_enabled:
log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', data.get('protocol'))
return False
# Download NZB or torrent file
filedata = None
if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))):
try:
filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id'))
except:
log.error('Tried to download, but the "%s" provider gave an error: %s', (data.get('protocol'), traceback.format_exc()))
return False
if filedata == 'try_next':
return filedata
elif not filedata:
return False
# Send NZB or torrent file to downloader
download_result = fireEvent('download', data = data, media = media, manual = manual, filedata = filedata, single = True)
if not download_result:
log.info('Tried to download, but the "%s" downloader gave an error', data.get('protocol'))
return False
log.debug('Downloader result: %s', download_result)
try:
db = get_db()
try:
rls = db.get('release_identifier', md5(data['url']), with_doc = True)['doc']
except:
log.error('No release found to store download information in')
return False
renamer_enabled = Env.setting('enabled', 'renamer')
# Save download-id info if returned
if isinstance(download_result, dict):
rls['download_info'] = download_result
db.update(rls)
log_movie = '%s (%s) in %s' % (getTitle(media), media['info'].get('year'), rls['quality'])
snatch_message = 'Snatched "%s": %s from %s' % (data.get('name'), log_movie, (data.get('provider', '') + data.get('provider_extra', '')))
log.info(snatch_message)
fireEvent('%s.snatched' % data['type'], message = snatch_message, data = media)
# Mark release as snatched
if renamer_enabled:
self.updateStatus(rls['_id'], status = 'snatched')
# If renamer isn't used, mark media done if finished or release downloaded
else:
if media['status'] == 'active':
profile = db.get('id', media['profile_id'])
if fireEvent('quality.isfinish', {'identifier': rls['quality'], 'is_3d': rls.get('is_3d', False)}, profile, single = True):
log.info('Renamer disabled, marking media as finished: %s', log_movie)
# Mark release done
self.updateStatus(rls['_id'], status = 'done')
# Mark media done
fireEvent('media.restatus', media['_id'], single = True)
return True
# Assume release downloaded
self.updateStatus(rls['_id'], status = 'downloaded')
except:
log.error('Failed storing download status: %s', traceback.format_exc())
return False
return True
def correctRelease(self, nzb = None, media = None, quality = None, **kwargs):
if media.get('type') != 'movie': return
media_title = fireEvent('searcher.get_search_title', media, single = True)
imdb_results = kwargs.get('imdb_results', False)
retention = Env.setting('retention', section = 'nzb')
if nzb.get('seeders') is None and 0 < retention < nzb.get('age', 0):
log.info2('Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name']))
return False
# Check for required and ignored words
if not fireEvent('searcher.correct_words', nzb['name'], media, single = True):
return False
preferred_quality = quality if quality else fireEvent('quality.single', identifier = quality['identifier'], single = True)
# Contains lower quality string
contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True)
if contains_other and isinstance(contains_other, dict):
log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality'))
return False
# Contains lower quality string
if not fireEvent('searcher.correct_3d', nzb, preferred_quality = preferred_quality, single = True):
log.info2('Wrong: %s, %slooking for %s in 3D', (nzb['name'], ('' if preferred_quality['custom'].get('3d') else 'NOT '), quality['label']))
return False
# File to small
if nzb['size'] and tryInt(preferred_quality['size_min']) > tryInt(nzb['size']):
log.info2('Wrong: "%s" is too small to be %s. %sMB instead of the minimal of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min']))
return False
# File to large
if nzb['size'] and tryInt(preferred_quality['size_max']) < tryInt(nzb['size']):
log.info2('Wrong: "%s" is too large to be %s. %sMB instead of the maximum of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max']))
return False
# Provider specific functions
get_more = nzb.get('get_more_info')
if get_more:
get_more(nzb)
extra_check = nzb.get('extra_check')
if extra_check and not extra_check(nzb):
return False
if imdb_results:
return True
# Check if nzb contains imdb link
if getImdb(nzb.get('description', '')) == getIdentifier(media):
return True
for raw_title in media['info']['titles']:
for movie_title in possibleTitles(raw_title):
movie_words = re.split('\W+', simplifyString(movie_title))
if fireEvent('searcher.correct_name', nzb['name'], movie_title, single = True):
# if no IMDB link, at least check year range 1
if len(movie_words) > 2 and fireEvent('searcher.correct_year', nzb['name'], media['info']['year'], 1, single = True):
return True
# if no IMDB link, at least check year
if len(movie_words) <= 2 and fireEvent('searcher.correct_year', nzb['name'], media['info']['year'], 0, single = True):
return True
log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], media_title, media['info']['year']))
return False
def conf(self, attr, value = None, default = None, section = None):
class_name = self.getName().lower().split(':')[0].lower()
return Env.setting(attr, section = section if section else class_name, value = value, default = default)
def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, stream = False):
url = quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
if not headers: headers = {}
if not data: data = {}
# Fill in some headers
parsed_url = urlparse(url)
host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else ''))
headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host))
headers['Host'] = headers.get('Host', None)
headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
headers['Connection'] = headers.get('Connection', 'keep-alive')
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
use_proxy = Env.setting('use_proxy')
proxy_url = None
if use_proxy:
proxy_server = Env.setting('proxy_server')
proxy_username = Env.setting('proxy_username')
proxy_password = Env.setting('proxy_password')
if proxy_server:
loc = "{0}:{1}@{2}".format(proxy_username, proxy_password, proxy_server) if proxy_username else proxy_server
proxy_url = {
"http": "http://"+loc,
"https": "https://"+loc,
}
else:
proxy_url = getproxies()
r = Env.get('http_opener')
# Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900):
log.info2('Disabled calls to %s for 15 minutes because so many failed requests.', host)
if not show_error:
raise Exception('Disabled calls to %s for 15 minutes because so many failed requests' % host)
else:
return ''
else:
del self.http_failed_request[host]
del self.http_failed_disabled[host]
self.wait(host, url)
status_code = None
try:
kwargs = {
'headers': headers,
'data': data if len(data) > 0 else None,
'timeout': timeout,
'files': files,
'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates..
'stream': stream,
'proxies': proxy_url,
}
method = 'post' if len(data) > 0 or files else 'get'
log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance(data, dict) else 'with data'))
response = r.request(method, url, **kwargs)
status_code = response.status_code
if response.status_code == requests.codes.ok:
data = response if stream else response.content
else:
response.raise_for_status()
self.http_failed_request[host] = 0
except (IOError, MaxRetryError, Timeout):
if show_error:
log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(0)))
# Save failed requests by hosts
try:
# To many requests
if status_code in [429]:
self.http_failed_request[host] = 1
self.http_failed_disabled[host] = time.time()
if not self.http_failed_request.get(host):
self.http_failed_request[host] = 1
else:
self.http_failed_request[host] += 1
# Disable temporarily
if self.http_failed_request[host] > 5 and not isLocalIP(host):
self.http_failed_disabled[host] = time.time()
except:
log.debug('Failed logging failed requests for %s: %s', (url, traceback.format_exc()))
raise
self.http_last_use[host] = time.time()
return data
