def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
self.options = getOptions(base_path, sys.argv[1:])
# Load settings
settings = Env.get('settings')
settings.setFile(self.options.config_file)
# Create data dir if needed
self.data_dir = os.path.expanduser(Env.setting('data_dir'))
if self.data_dir == '':
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, 'logs');
if not os.path.isdir(self.log_dir):
os.mkdir(self.log_dir)
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
from couchpotato.core.helpers.variable import getDataDir
self.options = getOptions(base_path, sys.argv[1:])
self.data_dir = getDataDir()
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
if self.options.daemon:
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
hdlr = handlers.RotatingFileHandler(os.path.join(self.data_dir, 'logs', 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
self.options = getOptions(sys.argv[1:])
# Load settings
settings = Env.get("settings")
settings.setFile(self.options.config_file)
# Create data dir if needed
if self.options.data_dir:
self.data_dir = self.options.data_dir
else:
self.data_dir = os.path.expanduser(Env.setting("data_dir"))
if self.data_dir == "":
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, "logs")
if not os.path.isdir(self.log_dir):
os.makedirs(self.log_dir)
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S")
hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, "error.log"), "a", 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
from bencode import bencode, bdecode
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt, ss
from couchpotato.core.logger import CPLog
from hashlib import sha1
from multipartpost import MultipartPostHandler
import cookielib
import httplib
import re
import time
import urllib
import urllib2
log = CPLog(__name__)
class uTorrent(Downloader):
type = ["torrent", "torrent_magnet"]
utorrent_api = None
def download(self, data, movie, filedata=None):
log.debug('Sending "%s" (%s) to uTorrent.', (data.get("name"), data.get("type")))
# Load host from config and split out port.
host = self.conf("host").split(":")
if not isInt(host[1]):
log.error("Config properties are not filled in correctly, port is missing.")
return False
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.logger import CPLog
import json
import requests
log = CPLog(__name__)
class Synology(Downloader):
type = ['nzb', 'torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data, movie, filedata = None):
response = False
log.error('Sending "%s" (%s) to Synology.', (data['name'], data['type']))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
try:
# Send request to Synology
srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password'))
if data['type'] == 'torrent_magnet':
log.info('Adding torrent URL %s', data['url'])
response = srpc.create_task(url = data['url'])
getImdb, link, symlink, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release, \
ReleaseInfo
from couchpotato.environment import Env
from unrar2 import RarFile
import errno
import fnmatch
import os
import re
import shutil
import time
import traceback
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
checking_snatched = False
def __init__(self):
addApiView('renamer.scan', self.scanView, docs = {
'desc': 'For the renamer to check for new files to rename in a folder',
'params': {
'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'},
'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'},
'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'},
'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'},
},
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.request import jsonified
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release
from couchpotato.environment import Env
import errno
import os
import re
import shutil
import traceback
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
def __init__(self):
addApiView('renamer.scan', self.scanView, docs = {
'desc': 'For the renamer to check for new files to rename',
})
addEvent('renamer.scan', self.scan)
addEvent('renamer.check_snatched', self.checkSnatched)
from base64 import b64encode
from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList
from couchpotato.core.helpers.encoding import isInt, sp
from couchpotato.core.helpers.variable import tryInt, tryFloat
from couchpotato.core.logger import CPLog
from datetime import timedelta
import httplib
import json
import os.path
import re
import urllib2
log = CPLog(__name__)
class Transmission(Downloader):
protocol = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
trpc = None
def connect(self):
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
if not self.trpc:
self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url'), username = self.conf('username'), password = self.conf('password'))
class YGG(TorrentProvider, MovieProvider):
"""
Couchpotato plugin to search movies torrents on www.yggtorrent.com.
.. seealso:: YarrProvider.login, Plugin.wait
"""
url_scheme = 'https'
domain_name = 'yggtorrent.is'
limit = 50
http_time_between_calls = 0
log = CPLog(__name__)
def __init__(self):
"""
Default constructor
"""
TorrentProvider.__init__(self)
MovieProvider.__init__(self)
path_www = YGG.url_scheme + '://' + YGG.domain_name
self.urls = {
'login': path_www + '/user/login',
'login_check': path_www + '/user/account',
'search': path_www + '/engine/search?{0}',
'torrent': path_www + '/torrent',
'url': path_www + '/engine/download_torrent?id={0}'
}
def getLoginParams(self):
"""
Return YGG login parameters.
.. seealso:: YarrProvider.getLoginParams
"""
return {
'id': self.conf('username'),
'pass': self.conf('password')
}
def loginSuccess(self, output):
"""
Check server's response on authentication.
.. seealso:: YarrProvider.loginSuccess
"""
return len(output) == 0
def loginCheckSuccess(self, output):
"""
Check if we are still connected.
.. seealso:: YarrProvider.loginCheckSuccess
"""
result = False
soup = BeautifulSoup(output, 'html.parser')
if soup.find(text=u'Déconnexion'):
result = True
return result
def getMoreInfo(self, nzb):
"""
Get details about a torrent.
.. seealso:: MovieSearcher.correctRelease
"""
data = self.getHTMLData(nzb['detail_url'])
soup = BeautifulSoup(data, 'html.parser')
description = soup.find(class_='description-header').find_next('div')
if description:
nzb['description'] = description.prettify()
line = soup.find(text=u'Uploadé le').find_next('td')
added = datetime.strptime(line.getText().split('(')[0].strip(),
'%d/%m/%Y %H:%M')
nzb['age'] = (datetime.now() - added).days
self.log.debug(nzb['age'])
def extraCheck(self, nzb):
"""
Exclusion when movie's description contains more than one IMDB
reference to prevent a movie bundle downloading. CouchPotato
is not able to extract a specific movie from an archive.
.. seealso:: MovieSearcher.correctRelease
"""
result = True
ids = getImdb(nzb.get('description', ''), multiple=True)
if len(ids) not in [0, 1]:
YGG.log.info('Too much IMDB ids: {0}'.format(', '.join(ids)))
result = False
return result
def parseText(self, node):
"""
Retrieve the text content from a HTML node.
"""
return node.getText().strip()
def _searchOnTitle(self, title, media, quality, results, offset=0):
"""
Do a search based on possible titles. This function doesn't check
the quality because CouchPotato do the job when parsing results.
Furthermore the URL must stay generic to use native CouchPotato
caching feature.
.. seealso:: YarrProvider.search
"""
try:
params = {
'category': 2145, # Film/Vidéo
'description': '',
'do': 'search',
'file': '',
'name': simplifyString(title),
'sub_category': 'all',
'uploader': ''
}
if offset > 0:
params['page'] = offset * YGG.limit
url = self.urls['search'].format(tryUrlencode(params))
data = self.getHTMLData(url)
soup = BeautifulSoup(data, 'html.parser')
filter_ = '^{0}'.format(self.urls['torrent'])
for link in soup.find_all(href=re.compile(filter_)):
detail_url = link['href']
if re.search(u'/filmvidéo/(film|animation|documentaire)/',
detail_url):
name = self.parseText(link)
id_ = tryInt(re.search('/(\d+)-[^/\s]+$', link['href']).
group(1))
columns = link.parent.parent.find_all('td')
size = self.parseSize(self.parseText(columns[5]))
seeders = tryInt(self.parseText(columns[7]))
leechers = tryInt(self.parseText(columns[8]))
result = {
'id': id_,
'name': name,
'seeders': seeders,
'leechers': leechers,
'size': size,
'url': self.urls['url'].format(id_),
'detail_url': detail_url,
'verified': True,
'get_more_info': self.getMoreInfo,
'extra_check': self.extraCheck
}
results.append(result)
YGG.log.debug(result)
# Get next page if we don't have all results
pagination = soup.find('ul', class_='pagination')
if pagination:
for page in pagination.find_all('li'):
next_ = tryInt(self.parseText(page.find('a')))
if next_ > offset + 1:
self._searchOnTitle(title, media, quality, results,
offset + 1)
break
except:
YGG.log.error('Failed searching release from {0}: {1}'.
format(self.getName(), traceback.format_exc()))
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
os.remove(os.path.join(backup_path, backup_file))
else:
existing_backups.append((int(ints[0]), backup_file))
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value = api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application(
[],
log_function = lambda x: None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order = True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def test():
fireEvent('app.shutdown')
add_reload_hook(test)
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application, no_keep_alive = True, ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive = True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView(
'settings',
self.view,
docs={
'desc':
'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.',
'return': {
'type':
'object',
'example':
"""{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}"""
}
})
addApiView('settings.save',
self.saveView,
docs={
'desc': 'Save setting to config file (settings.conf)',
'params': {
'section': {
'desc': 'The section name in settings.conf'
},
'name': {
'desc': 'The option name'
},
'value': {
'desc': 'The value you want to save'
},
}
})
addEvent('database.setup', self.databaseSetup)
self.file = None
self.p = None
self.log = None
self.directories_delimiter = "::"
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def databaseSetup(self):
fireEvent('database.setup_index', 'property', PropertyIndex)
def parser(self):
return self.p
def sections(self):
res = filter(self.isSectionReadable, self.p.sections())
return res
def connectEvents(self):
addEvent('settings.options', self.addOptions)
addEvent('settings.register', self.registerDefaults)
addEvent('settings.save', self.save)
def registerDefaults(self, section_name, options=None, save=True):
if not options: options = {}
self.addSection(section_name)
for option_name, option in options.items():
self.setDefault(section_name, option_name,
option.get('default', ''))
# Set UI-meta for option (hidden/ro/rw)
if option.get('ui-meta'):
value = option.get('ui-meta')
if value:
value = value.lower()
if value in ['hidden', 'rw', 'ro']:
meta_option_name = option_name + self.optionMetaSuffix(
)
self.setDefault(section_name, meta_option_name, value)
else:
self.log.warning(
'Wrong value for option %s.%s : ui-meta can not be equal to "%s"',
(section_name, option_name, value))
# Migrate old settings from old location to the new location
if option.get('migrate_from'):
if self.p.has_option(option.get('migrate_from'), option_name):
previous_value = self.p.get(option.get('migrate_from'),
option_name)
self.p.set(section_name, option_name, previous_value)
self.p.remove_option(option.get('migrate_from'),
option_name)
if option.get('type'):
self.setType(section_name, option_name, option.get('type'))
if save:
self.save()
def set(self, section, option, value):
if not self.isOptionWritable(section, option):
self.log.warning('set::option "%s.%s" isn\'t writable',
(section, option))
return None
if self.isOptionMeta(section, option):
self.log.warning(
'set::option "%s.%s" cancelled, since it is a META option',
(section, option))
return None
return self.p.set(section, option, value)
def get(self, option='', section='core', default=None, type=None):
if self.isOptionMeta(section, option):
self.log.warning(
'get::option "%s.%s" cancelled, since it is a META option',
(section, option))
return None
tp = type
try:
tp = self.getType(section, option) if not tp else tp
if hasattr(self, 'get%s' % tp.capitalize()):
return getattr(self, 'get%s' % tp.capitalize())(section,
option)
else:
return self.getUnicode(section, option)
except:
return default
def delete(self, option='', section='core'):
if not self.isOptionWritable(section, option):
self.log.warning('delete::option "%s.%s" isn\'t writable',
(section, option))
return None
if self.isOptionMeta(section, option):
self.log.warning(
'set::option "%s.%s" cancelled, since it is a META option',
(section, option))
return None
self.p.remove_option(section, option)
self.save()
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option) == 1
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryFloat(self.p.get(section, option))
def getDirectories(self, section, option):
value = self.p.get(section, option)
if value:
return map(str.strip, str.split(value, self.directories_delimiter))
return []
def getUnicode(self, section, option):
value = self.p.get(section, option).decode('unicode_escape')
return toUnicode(value).strip()
def getValues(self):
from couchpotato.environment import Env
values = {}
soft_chroot = Env.get('softchroot')
# TODO : There is two commented "continue" blocks (# COMMENTED_SKIPPING). They both are good...
# ... but, they omit output of values of hidden and non-readable options
# Currently, such behaviour could break the Web UI of CP...
# So, currently this two blocks are commented (but they are required to
# provide secure hidding of options.
for section in self.sections():
# COMMENTED_SKIPPING
#if not self.isSectionReadable(section):
# continue
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
#skip meta options:
if self.isOptionMeta(section, option_name):
continue
# COMMENTED_SKIPPING
#if not self.isOptionReadable(section, option_name):
# continue
value = self.get(option_name, section)
is_password = self.getType(section, option_name) == 'password'
if is_password and value:
value = len(value) * '*'
# chrootify directory before sending to UI:
if (self.getType(section, option_name)
== 'directory') and value:
try:
value = soft_chroot.abs2chroot(value)
except:
value = ""
# chrootify directories before sending to UI:
if (self.getType(section, option_name) == 'directories'):
if (not value):
value = []
try:
value = map(soft_chroot.abs2chroot, value)
except:
value = []
values[section][option_name] = value
return values
def save(self):
with open(self.file, 'wb') as configfile:
self.p.write(configfile)
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def getType(self, section, option):
tp = None
try:
tp = self.types[section][option]
except:
tp = 'unicode' if not tp else tp
return tp
def addOptions(self, section_name, options):
# no additional actions (related to ro-rw options) are required here
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name],
options)
def getOptions(self):
"""Returns dict of UI-readable options
To check, whether the option is readable self.isOptionReadable() is used
"""
res = {}
# it is required to filter invisible options for UI, but also we should
# preserve original tree for server's purposes.
# So, next loops do one thing: copy options to res and in the process
# 1. omit NON-READABLE (for UI) options, and
# 2. put flags on READONLY options
for section_key in self.options.keys():
section_orig = self.options[section_key]
section_name = section_orig.get(
'name') if 'name' in section_orig else section_key
if self.isSectionReadable(section_name):
section_copy = {}
section_copy_groups = []
for section_field in section_orig:
if section_field.lower() != 'groups':
section_copy[section_field] = section_orig[
section_field]
else:
for group_orig in section_orig['groups']:
group_copy = {}
group_copy_options = []
for group_field in group_orig:
if group_field.lower() != 'options':
group_copy[group_field] = group_orig[
group_field]
else:
for option in group_orig[group_field]:
option_name = option.get('name')
# You should keep in mind, that READONLY = !IS_WRITABLE
# and IS_READABLE is a different thing
if self.isOptionReadable(
section_name, option_name):
group_copy_options.append(option)
if not self.isOptionWritable(
section_name, option_name):
option['readonly'] = True
if len(group_copy_options) > 0:
group_copy['options'] = group_copy_options
section_copy_groups.append(group_copy)
if len(section_copy_groups) > 0:
section_copy['groups'] = section_copy_groups
res[section_key] = section_copy
return res
def view(self, **kwargs):
return {'options': self.getOptions(), 'values': self.getValues()}
def saveView(self, **kwargs):
section = kwargs.get('section')
option = kwargs.get('name')
value = kwargs.get('value')
if not self.isOptionWritable(section, option):
self.log.warning('Option "%s.%s" isn\'t writable',
(section, option))
return {
'success': False,
}
from couchpotato.environment import Env
soft_chroot = Env.get('softchroot')
if self.getType(section, option) == 'directory':
value = soft_chroot.chroot2abs(value)
if self.getType(section, option) == 'directories':
import json
value = json.loads(value)
if not (value and isinstance(value, list)):
value = []
value = map(soft_chroot.chroot2abs, value)
value = self.directories_delimiter.join(value)
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option),
value,
single=True)
self.set(section, option,
(new_value if new_value else value).encode('unicode_escape'))
self.save()
# After save (for re-interval etc)
fireEvent('setting.save.%s.%s.after' % (section, option), single=True)
fireEvent('setting.save.%s.*.after' % section, single=True)
return {'success': True}
def isSectionReadable(self, section):
meta = 'section_hidden' + self.optionMetaSuffix()
try:
return not self.p.getboolean(section, meta)
except:
pass
# by default - every section is readable:
return True
def isOptionReadable(self, section, option):
meta = option + self.optionMetaSuffix()
if self.p.has_option(section, meta):
meta_v = self.p.get(section, meta).lower()
return (meta_v == 'rw') or (meta_v == 'ro')
# by default - all is writable:
return True
def optionReadableCheckAndWarn(self, section, option):
x = self.isOptionReadable(section, option)
if not x:
self.log.warning('Option "%s.%s" isn\'t readable',
(section, option))
return x
def isOptionWritable(self, section, option):
meta = option + self.optionMetaSuffix()
if self.p.has_option(section, meta):
return self.p.get(section, meta).lower() == 'rw'
# by default - all is writable:
return True
def optionMetaSuffix(self):
return '_internal_meta'
def isOptionMeta(self, section, option):
""" A helper method for detecting internal-meta options in the ini-file
For a meta options used following names:
* section_hidden_internal_meta = (True | False) - for section visibility
* <OPTION>_internal_meta = (ro|rw|hidden) - for section visibility
"""
suffix = self.optionMetaSuffix()
return option.endswith(suffix)
def getProperty(self, identifier):
from couchpotato import get_db
db = get_db()
prop = None
identifier = identifier.encode(
"ascii", "ignore"
) # if identifier is not ascii it crashes below in the db access
try:
propert = db.get('property', identifier, with_doc=True)
prop = propert['doc']['value']
except ValueError:
propert = db.get('property', identifier)
fireEvent('database.delete_corrupted', propert.get('_id'))
except:
self.log.debug('Property "%s" doesn\'t exist: %s',
(identifier, traceback.format_exc(0)))
return prop
def setProperty(self, identifier, value=''):
from couchpotato import get_db
db = get_db()
try:
p = db.get('property', identifier, with_doc=True)
p['doc'].update({
'identifier': identifier,
'value': toUnicode(value),
})
db.update(p['doc'])
except:
db.insert({
'_t': 'property',
'identifier': identifier,
'value': toUnicode(value),
})
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.request import jsonified
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release
from couchpotato.environment import Env
import errno
import os
import re
import shutil
import traceback
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
checking_snatched = False
def __init__(self):
addApiView('renamer.scan',
self.scanView,
docs={
'desc':
'For the renamer to check for new files to rename',
})
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import toUnicode, ss
from couchpotato.core.helpers.request import jsonified
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release
from couchpotato.environment import Env
import errno
import os
import re
import shutil
import traceback
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
checking_snatched = False
def __init__(self):
addApiView('renamer.scan',
self.scanView,
docs={
'desc':
'For the renamer to check for new files to rename',
})
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
# Do db stuff
db_path = os.path.join(data_dir, 'couchpotato.db')
# Backup before start and cleanup old databases
new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time())))
# Create path and copy
if not os.path.isdir(new_backup): os.makedirs(new_backup)
src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal']
for src_file in src_files:
if os.path.isfile(src_file):
shutil.copy2(src_file, os.path.join(new_backup, os.path.basename(src_file)))
# Remove older backups, keep backups 3 days or at least 3
backups = []
for directory in os.listdir(os.path.dirname(new_backup)):
backup = os.path.join(os.path.dirname(new_backup), directory)
if os.path.isdir(backup):
backups.append(backup)
total_backups = len(backups)
for backup in backups:
if total_backups > 3:
if int(os.path.basename(backup)) < time.time() - 259200:
for src_file in src_files:
b_file = os.path.join(backup, os.path.basename(src_file))
if os.path.isfile(b_file):
os.remove(b_file)
os.rmdir(backup)
total_backups -= 1
# Register environment settings
Env.set('encoding', encoding)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + db_path)
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = base_path)
loader.run()
# Load migrations
initialize = True
db = Env.get('db_path')
if os.path.isfile(db_path):
initialize = False
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version = latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
if initialize:
fireEvent('app.initialize', in_order = True)
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else ''
# Basic config
app.secret_key = api_key
# app.debug = development
config = {
'use_reloader': reloader,
'host': Env.setting('host', default = '0.0.0.0'),
'port': tryInt(Env.setting('port', default = 5000))
}
# Static path
app.static_folder = os.path.join(base_path, 'couchpotato', 'static')
web.add_url_rule('api/%s/static/<path:filename>' % api_key,
endpoint = 'static',
view_func = app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix = '%s/' % url_base)
app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key))
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
# Go go go!
web_container = WSGIContainer(app)
web_container._log = _log
loop = IOLoop.instance()
application = Application([
(r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler),
(r'.*', FallbackHandler, dict(fallback = web_container)),
],
log_function = lambda x : None,
debug = config['use_reloader']
)
try_restart = True
restart_tries = 5
while try_restart:
try:
application.listen(config['port'], config['host'], no_keep_alive = True)
loop.start()
except Exception, e:
try:
nr, msg = e
if nr == 48:
log.info('Already in use, try %s more time after few seconds', restart_tries)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
from couchpotato.core.helpers.variable import splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification
from urllib2 import URLError
import base64
import json
import socket
import traceback
import urllib
log = CPLog(__name__)
class XBMC(Notification):
listen_to = ['renamer.after', 'movie.snatched']
use_json_notifications = {}
http_time_between_calls = 0
def notify(self, message='', data=None, listener=None):
if not data: data = {}
hosts = splitString(self.conf('host'))
successful = 0
max_successful = 0
for host in hosts:
if self.use_json_notifications.get(host) is None:
self.getXBMCJSONversion(host, message=message)
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.logger import CPLog
import json
import requests
log = CPLog(__name__)
class Synology(Downloader):
protocol = ['nzb', 'torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data = None, movie = None, filedata = None):
if not movie: movie = {}
if not data: data = {}
response = False
log.error('Sending "%s" (%s) to Synology.', (data['name'], data['protocol']))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
try:
# Send request to Synology
srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password'))
if data['protocol'] == 'torrent_magnet':
from couchpotato.core.helpers.variable import tryInt, tryFloat
from couchpotato.core.logger import CPLog
from datetime import timedelta
from hashlib import sha1
from multipartpost import MultipartPostHandler
import cookielib
import httplib
import json
import os
import re
import stat
import time
import urllib
import urllib2
log = CPLog(__name__)
class uTorrent(Downloader):
protocol = ['torrent', 'torrent_magnet']
utorrent_api = None
def connect(self):
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password'))
class Deluge(DownloaderBase):
protocol = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
drpc = None
def connect(self, reconnect=False):
""" Connect to the delugeRPC, re-use connection when already available
:param reconnect: force reconnect
:return: DelugeRPC instance
"""
# Load host from config and split out port.
host = cleanHost(self.conf('host'), protocol=False).split(':')
# Force host assignment
if len(host) == 1:
host.append(80)
if not isInt(host[1]):
log.error(
'Config properties are not filled in correctly, port is missing.'
)
return False
if not self.drpc or reconnect:
self.drpc = DelugeRPC(host[0],
port=host[1],
username=self.conf('username'),
password=self.conf('password'))
return self.drpc
def download(self, data=None, media=None, filedata=None):
""" Send a torrent/nzb file to the downloader
:param data: dict returned from provider
Contains the release information
:param media: media dict with information
Used for creating the filename when possible
:param filedata: downloaded torrent/nzb filedata
The file gets downloaded in the searcher and send to this function
This is done to have failed checking before using the downloader, so the downloader
doesn't need to worry about that
:return: boolean
One faile returns false, but the downloaded should log his own errors
"""
if not media: media = {}
if not data: data = {}
log.info('Sending "%s" (%s) to Deluge.',
(data.get('name'), data.get('protocol')))
if not self.connect():
return False
if not filedata and data.get('protocol') == 'torrent':
log.error('Failed sending torrent, no data')
return False
# Set parameters for Deluge
options = {
'add_paused': self.conf('paused', default=0),
'label': self.conf('label')
}
if self.conf('directory'):
#if os.path.isdir(self.conf('directory')):
options['download_location'] = self.conf('directory')
#else:
# log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory'))
if self.conf('completed_directory'):
#if os.path.isdir(self.conf('completed_directory')):
options['move_completed'] = 1
options['move_completed_path'] = self.conf('completed_directory')
#else:
# log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory'))
if data.get('seed_ratio'):
options['stop_at_ratio'] = 1
options['stop_ratio'] = tryFloat(data.get('seed_ratio'))
# Deluge only has seed time as a global option. Might be added in
# in a future API release.
# if data.get('seed_time'):
# Send request to Deluge
if data.get('protocol') == 'torrent_magnet':
remote_torrent = self.drpc.add_torrent_magnet(
data.get('url'), options)
else:
filename = self.createFileName(data, filedata, media)
remote_torrent = self.drpc.add_torrent_file(
filename, filedata, options)
if not remote_torrent:
log.error('Failed sending torrent to Deluge')
return False
log.info('Torrent sent to Deluge successfully.')
return self.downloadReturnId(remote_torrent)
def test(self):
""" Check if connection works
:return: bool
"""
if self.connect(True) and self.drpc.test():
return True
return False
def getAllDownloadStatus(self, ids):
""" Get status of all active downloads
:param ids: list of (mixed) downloader ids
Used to match the releases for this downloader as there could be
other downloaders active that it should ignore
:return: list of releases
"""
log.debug('Checking Deluge download status.')
if not self.connect():
return []
release_downloads = ReleaseDownloadList(self)
queue = self.drpc.get_alltorrents(ids)
if not queue:
log.debug('Nothing in queue or error')
return []
for torrent_id in queue:
torrent = queue[torrent_id]
if not 'hash' in torrent:
# When given a list of ids, deluge will return an empty item for a non-existant torrent.
continue
log.debug(
'name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s',
(torrent['name'], torrent['hash'], torrent['save_path'],
torrent['move_on_completed'], torrent['move_completed_path'],
torrent['hash'], torrent['progress'], torrent['state'],
torrent['eta'], torrent['ratio'], torrent['stop_ratio'],
torrent['is_seed'], torrent['is_finished'],
torrent['paused']))
# Deluge has no easy way to work out if a torrent is stalled or failing.
#status = 'failed'
status = 'busy'
# If an user opts to seed a torrent forever (usually associated to private trackers usage), stop_ratio will be 0 or -1 (depending on Deluge version).
# In this scenario the status of the torrent would never change from BUSY to SEEDING.
# The last check takes care of this case.
if torrent['is_seed'] and (
(tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio']))
or (tryFloat(torrent['stop_ratio']) < 0)):
# We have torrent['seeding_time'] to work out what the seeding time is, but we do not
# have access to the downloader seed_time, as with deluge we have no way to pass it
# when the torrent is added. So Deluge will only look at the ratio.
# See above comment in download().
status = 'seeding'
elif torrent['is_seed'] and torrent['is_finished'] and torrent[
'paused'] and torrent['state'] == 'Paused':
status = 'completed'
download_dir = sp(torrent['save_path'])
if torrent['move_on_completed']:
download_dir = torrent['move_completed_path']
torrent_files = []
for file_item in torrent['files']:
torrent_files.append(
sp(os.path.join(download_dir, file_item['path'])))
release_downloads.append({
'id':
torrent['hash'],
'name':
torrent['name'],
'status':
status,
'original_status':
torrent['state'],
'seed_ratio':
torrent['ratio'],
'timeleft':
str(timedelta(seconds=torrent['eta'])),
'folder':
sp(download_dir if len(torrent_files) ==
1 else os.path.join(download_dir, torrent['name'])),
'files':
torrent_files,
})
return release_downloads
def pause(self, release_download, pause=True):
if pause:
return self.drpc.pause_torrent([release_download['id']])
else:
return self.drpc.resume_torrent([release_download['id']])
def removeFailed(self, release_download):
log.info('%s failed downloading, deleting...',
release_download['name'])
return self.drpc.remove_torrent(release_download['id'], True)
def processComplete(self, release_download, delete_files=False):
log.debug(
'Requesting Deluge to remove the torrent %s%s.',
(release_download['name'],
' and cleanup the downloaded files' if delete_files else ''))
return self.drpc.remove_torrent(release_download['id'],
remove_local_data=delete_files)
def runCouchPotato(options, base_path, args, desktop = None):
# Load settings
from couchpotato.environment import Env
settings = Env.get('settings')
settings.setFile(options.config_file)
# Create data dir if needed
data_dir = os.path.expanduser(Env.setting('data_dir'))
if data_dir == '':
data_dir = getDataDir()
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
# Create logging dir
log_dir = os.path.join(data_dir, 'logs');
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
pass
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
# Register environment settings
Env.set('encoding', encoding)
Env.set('uses_git', not options.nogit)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db'))
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Disable server access log
server_log = logging.getLogger('werkzeug')
server_log.disabled = True
# Only run once when debugging
fire_load = False
if os.environ.get('WERKZEUG_RUN_MAIN') or not debug or Env.get('desktop') or options.daemon:
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s' % options)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = base_path)
loader.run()
# Load migrations
from migrate.versioning.api import version_control, db_version, version, upgrade
db = Env.get('db_path')
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
logging.getLogger('migrate').setLevel(logging.WARNING) # Disable logging for migration
latest_db_version = version(repo)
initialize = True
try:
current_db_version = db_version(db, repo)
initialize = False
except:
version_control(db, repo, version = latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
if initialize:
fireEvent('app.initialize')
fire_load = True
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else ''
reloader = debug is True and not Env.get('desktop') and not options.daemon
# Basic config
app.secret_key = api_key
config = {
'use_reloader': reloader,
'host': Env.setting('host', default = '0.0.0.0'),
'port': tryInt(Env.setting('port', default = 5000))
}
# Static path
web.add_url_rule('static/<path:filename>',
endpoint = 'static',
view_func = app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix = '%s/' % url_base)
app.register_blueprint(api, url_prefix = '%s/%s/' % (url_base, api_key))
# Some logging and fire load event
try: log.info('Starting server on port %(port)s' % config)
except: pass
if fire_load: fireEventAsync('app.load')
# Go go go!
try:
app.run(**config)
except (KeyboardInterrupt, SystemExit):
raise
except:
log.error('Failed starting: %s' % traceback.format_exc())
raise
from apscheduler.scheduler import Scheduler as Sched
from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
import logging
log = CPLog(__name__)
class Scheduler(Plugin):
crons = {}
intervals = {}
started = False
def __init__(self):
addEvent('schedule.cron', self.cron)
addEvent('schedule.interval', self.interval)
addEvent('schedule.start', self.start)
addEvent('schedule.restart', self.start)
addEvent('app.load', self.start)
self.sched = Sched(misfire_grace_time = 60)
def remove(self, identifier):
for type in ['interval', 'cron']:
try:
self.sched.unschedule_job(getattr(self, type)[identifier]['job'])
log.debug('%s unscheduled %s', (type.capitalize(), identifier))
class Loader(object):
do_restart = False
def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
self.options = getOptions(base_path, sys.argv[1:])
# Load settings
settings = Env.get('settings')
settings.setFile(self.options.config_file)
# Create data dir if needed
self.data_dir = os.path.expanduser(Env.setting('data_dir'))
if self.data_dir == '':
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, 'logs');
if not os.path.isdir(self.log_dir):
os.mkdir(self.log_dir)
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def addSignals(self):
signal.signal(signal.SIGINT, self.onExit)
signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(1))
from couchpotato.core.event import addEvent
addEvent('app.after_shutdown', self.afterShutdown)
def afterShutdown(self, restart):
self.do_restart = restart
def onExit(self, signal, frame):
from couchpotato.core.event import fireEvent
fireEvent('app.crappy_shutdown', single = True)
def run(self):
self.addSignals()
from couchpotato.runner import runCouchPotato
runCouchPotato(self.options, base_path, sys.argv[1:], data_dir = self.data_dir, log_dir = self.log_dir, Env = Env)
if self.do_restart:
self.restart()
def restart(self):
try:
# remove old pidfile first
try:
if self.runAsDaemon():
try: self.daemon.stop()
except: pass
except:
self.log.critical(traceback.format_exc())
args = [sys.executable] + [os.path.join(base_path, __file__)] + sys.argv[1:]
subprocess.Popen(args)
except:
self.log.critical(traceback.format_exc())
def daemonize(self):
if self.runAsDaemon():
try:
from daemon import Daemon
self.daemon = Daemon(self.options.pid_file)
self.daemon.daemonize()
except SystemExit:
raise
except:
self.log.critical(traceback.format_exc())
def runAsDaemon(self):
return self.options.daemon and self.options.pid_file
class Deluge(Downloader):
protocol = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
drpc = None
def connect(self, reconnect=False):
# Load host from config and split out port.
host = cleanHost(self.conf('host'), protocol=False).split(':')
if not isInt(host[1]):
log.error(
'Config properties are not filled in correctly, port is missing.'
)
return False
if not self.drpc or reconnect:
self.drpc = DelugeRPC(host[0],
port=host[1],
username=self.conf('username'),
password=self.conf('password'))
return self.drpc
def download(self, data=None, media=None, filedata=None):
if not media: media = {}
if not data: data = {}
log.info('Sending "%s" (%s) to Deluge.',
(data.get('name'), data.get('protocol')))
if not self.connect():
return False
if not filedata and data.get('protocol') == 'torrent':
log.error('Failed sending torrent, no data')
return False
# Set parameters for Deluge
options = {
'add_paused': self.conf('paused', default=0),
'label': self.conf('label')
}
if self.conf('directory'):
if os.path.isdir(self.conf('directory')):
options['download_location'] = self.conf('directory')
else:
log.error(
'Download directory from Deluge settings: %s doesn\'t exist',
self.conf('directory'))
if self.conf('completed_directory'):
if os.path.isdir(self.conf('completed_directory')):
options['move_completed'] = 1
options['move_completed_path'] = self.conf(
'completed_directory')
else:
log.error(
'Download directory from Deluge settings: %s doesn\'t exist',
self.conf('directory'))
if data.get('seed_ratio'):
options['stop_at_ratio'] = 1
options['stop_ratio'] = tryFloat(data.get('seed_ratio'))
# Deluge only has seed time as a global option. Might be added in
# in a future API release.
# if data.get('seed_time'):
# Send request to Deluge
if data.get('protocol') == 'torrent_magnet':
remote_torrent = self.drpc.add_torrent_magnet(
data.get('url'), options)
else:
filename = self.createFileName(data, filedata, media)
remote_torrent = self.drpc.add_torrent_file(
filename, filedata, options)
if not remote_torrent:
log.error('Failed sending torrent to Deluge')
return False
log.info('Torrent sent to Deluge successfully.')
return self.downloadReturnId(remote_torrent)
def test(self):
if self.connect(True) and self.drpc.test():
return True
return False
def getAllDownloadStatus(self, ids):
log.debug('Checking Deluge download status.')
if not self.connect():
return []
release_downloads = ReleaseDownloadList(self)
queue = self.drpc.get_alltorrents(ids)
if not queue:
log.debug('Nothing in queue or error')
return []
for torrent_id in queue:
torrent = queue[torrent_id]
if not 'hash' in torrent:
# When given a list of ids, deluge will return an empty item for a non-existant torrent.
continue
log.debug(
'name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s',
(torrent['name'], torrent['hash'], torrent['save_path'],
torrent['move_on_completed'], torrent['move_completed_path'],
torrent['hash'], torrent['progress'], torrent['state'],
torrent['eta'], torrent['ratio'], torrent['stop_ratio'],
torrent['is_seed'], torrent['is_finished'],
torrent['paused']))
# Deluge has no easy way to work out if a torrent is stalled or failing.
#status = 'failed'
status = 'busy'
if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat(
torrent['stop_ratio']):
# We have torrent['seeding_time'] to work out what the seeding time is, but we do not
# have access to the downloader seed_time, as with deluge we have no way to pass it
# when the torrent is added. So Deluge will only look at the ratio.
# See above comment in download().
status = 'seeding'
elif torrent['is_seed'] and torrent['is_finished'] and torrent[
'paused'] and torrent['state'] == 'Paused':
status = 'completed'
download_dir = sp(torrent['save_path'])
if torrent['move_on_completed']:
download_dir = torrent['move_completed_path']
torrent_files = []
for file_item in torrent['files']:
torrent_files.append(
sp(os.path.join(download_dir, file_item['path'])))
release_downloads.append({
'id':
torrent['hash'],
'name':
torrent['name'],
'status':
status,
'original_status':
torrent['state'],
'seed_ratio':
torrent['ratio'],
'timeleft':
str(timedelta(seconds=torrent['eta'])),
'folder':
sp(download_dir if len(torrent_files) ==
1 else os.path.join(download_dir, torrent['name'])),
'files':
'|'.join(torrent_files),
})
return release_downloads
def pause(self, release_download, pause=True):
if pause:
return self.drpc.pause_torrent([release_download['id']])
else:
return self.drpc.resume_torrent([release_download['id']])
def removeFailed(self, release_download):
log.info('%s failed downloading, deleting...',
release_download['name'])
return self.drpc.remove_torrent(release_download['id'], True)
def processComplete(self, release_download, delete_files=False):
log.debug(
'Requesting Deluge to remove the torrent %s%s.',
(release_download['name'],
' and cleanup the downloaded files' if delete_files else ''))
return self.drpc.remove_torrent(release_download['id'],
remove_local_data=delete_files)
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db'))
# Backup before start and cleanup old databases
new_backup = toUnicode(os.path.join(data_dir, 'db_backup', str(int(time.time()))))
# Create path and copy
if not os.path.isdir(new_backup): os.makedirs(new_backup)
src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal']
for src_file in src_files:
if os.path.isfile(src_file):
dst_file = toUnicode(os.path.join(new_backup, os.path.basename(src_file)))
shutil.copyfile(src_file, dst_file)
# Try and copy stats seperately
try: shutil.copystat(src_file, dst_file)
except: pass
# Remove older backups, keep backups 3 days or at least 3
backups = []
for directory in os.listdir(os.path.dirname(new_backup)):
backup = toUnicode(os.path.join(os.path.dirname(new_backup), directory))
if os.path.isdir(backup):
backups.append(backup)
total_backups = len(backups)
for backup in backups:
if total_backups > 3:
if tryInt(os.path.basename(backup)) < time.time() - 259200:
for the_file in os.listdir(backup):
file_path = os.path.join(backup, the_file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
except:
raise
os.rmdir(backup)
total_backups -= 1
# Register environment settings
Env.set('app_dir', toUnicode(base_path))
Env.set('data_dir', toUnicode(data_dir))
Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db_path', toUnicode('sqlite:///' + db_path))
Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache')))
Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python'))))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Check if database exists
db = Env.get('db_path')
db_exists = os.path.isfile(toUnicode(db_path))
# Load migrations
if db_exists:
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version = latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version:
if development:
log.error('There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.')
else:
log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application([],
log_function = lambda x : None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = toUnicode(base_path))
loader.run()
# Fill database with needed stuff
if not db_exists:
fireEvent('app.initialize', in_order = True)
# Go go go!
from tornado.ioloop import IOLoop
loop = IOLoop.current()
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application, no_keep_alive = True, ssl_options = {
"certfile": config['ssl_cert'],
"keyfile": config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive = True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception, e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
def runCouchPotato(options,
base_path,
args,
data_dir=None,
log_dir=None,
Env=None,
desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db'))
# Backup before start and cleanup old databases
new_backup = toUnicode(
os.path.join(data_dir, 'db_backup', str(int(time.time()))))
if not os.path.isdir(new_backup): os.makedirs(new_backup)
# Remove older backups, keep backups 3 days or at least 3
backups = []
for directory in os.listdir(os.path.dirname(new_backup)):
backup = toUnicode(os.path.join(os.path.dirname(new_backup),
directory))
if os.path.isdir(backup):
backups.append(backup)
latest_backup = tryInt(os.path.basename(
sorted(backups)[-1])) if len(backups) > 0 else 0
if latest_backup < time.time() - 3600:
# Create path and copy
src_files = [
options.config_file, db_path, db_path + '-shm', db_path + '-wal'
]
for src_file in src_files:
if os.path.isfile(src_file):
dst_file = toUnicode(
os.path.join(new_backup, os.path.basename(src_file)))
shutil.copyfile(src_file, dst_file)
# Try and copy stats seperately
try:
shutil.copystat(src_file, dst_file)
except:
pass
total_backups = len(backups)
for backup in backups:
if total_backups > 3:
if tryInt(os.path.basename(backup)) < time.time() - 259200:
for the_file in os.listdir(backup):
file_path = os.path.join(backup, the_file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
except:
raise
os.rmdir(backup)
total_backups -= 1
# Register environment settings
Env.set('app_dir', toUnicode(base_path))
Env.set('data_dir', toUnicode(data_dir))
Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db_path', toUnicode('sqlite:///' + db_path))
Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache')))
Env.set(
'cache',
FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'),
'python'))))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default=False, type='bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default=False, type='bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get(
'desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or
options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Check if database exists
db = Env.get('db_path')
db_exists = os.path.isfile(toUnicode(db_path))
# Load migrations
if db_exists:
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version=latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version:
if development:
log.error(
'There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.'
)
else:
log.info('Doing database upgrade. From %d to %d',
(current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') +
'/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default='0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default=5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default=None),
'ssl_key': Env.setting('ssl_key', default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config['use_reloader'],
gzip=True,
cookie_secret=api_key,
login_url='%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(
".*$",
[
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {
"url": web_base + 'docs/'
}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(
".*$",
[('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {
'path':
toUnicode(
os.path.join(base_path, 'couchpotato', 'static', dir_name))
})])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=toUnicode(base_path))
loader.run()
# Fill database with needed stuff
if not db_exists:
fireEvent('app.initialize', in_order=True)
# Go go go!
from tornado.ioloop import IOLoop
loop = IOLoop.current()
# Some logging and fire load event
try:
log.info('Starting server on port %(port)s', config)
except:
pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application,
no_keep_alive=True,
ssl_options={
"certfile": config['ssl_cert'],
"keyfile": config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive=True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception, e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
'Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds',
(config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView(
"settings",
self.view,
docs={
"desc": "Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.",
"return": {
"type": "object",
"example": """{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}""",
},
},
)
addApiView(
"settings.save",
self.saveView,
docs={
"desc": "Save setting to config file (settings.conf)",
"params": {
"section": {"desc": "The section name in settings.conf"},
"name": {"desc": "The option name"},
"value": {"desc": "The value you want to save"},
},
},
)
addEvent("database.setup", self.databaseSetup)
self.file = None
self.p = None
self.log = None
self.directories_delimiter = "::"
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def databaseSetup(self):
fireEvent("database.setup_index", "property", PropertyIndex)
def parser(self):
return self.p
def sections(self):
res = filter(self.isSectionReadable, self.p.sections())
return res
def connectEvents(self):
addEvent("settings.options", self.addOptions)
addEvent("settings.register", self.registerDefaults)
addEvent("settings.save", self.save)
def registerDefaults(self, section_name, options=None, save=True):
if not options:
options = {}
self.addSection(section_name)
for option_name, option in options.items():
self.setDefault(section_name, option_name, option.get("default", ""))
# Set UI-meta for option (hidden/ro/rw)
if option.get("ui-meta"):
value = option.get("ui-meta")
if value:
value = value.lower()
if value in ["hidden", "rw", "ro"]:
meta_option_name = option_name + self.optionMetaSuffix()
self.setDefault(section_name, meta_option_name, value)
else:
self.log.warning(
'Wrong value for option %s.%s : ui-meta can not be equal to "%s"',
(section_name, option_name, value),
)
# Migrate old settings from old location to the new location
if option.get("migrate_from"):
if self.p.has_option(option.get("migrate_from"), option_name):
previous_value = self.p.get(option.get("migrate_from"), option_name)
self.p.set(section_name, option_name, previous_value)
self.p.remove_option(option.get("migrate_from"), option_name)
if option.get("type"):
self.setType(section_name, option_name, option.get("type"))
if save:
self.save()
def set(self, section, option, value):
if not self.isOptionWritable(section, option):
self.log.warning('set::option "%s.%s" isn\'t writable', (section, option))
return None
if self.isOptionMeta(section, option):
self.log.warning('set::option "%s.%s" cancelled, since it is a META option', (section, option))
return None
return self.p.set(section, option, value)
def get(self, option="", section="core", default=None, type=None):
if self.isOptionMeta(section, option):
self.log.warning('get::option "%s.%s" cancelled, since it is a META option', (section, option))
return None
tp = type
try:
tp = self.getType(section, option) if not tp else tp
if hasattr(self, "get%s" % tp.capitalize()):
return getattr(self, "get%s" % tp.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def delete(self, option="", section="core"):
if not self.isOptionWritable(section, option):
self.log.warning('delete::option "%s.%s" isn\'t writable', (section, option))
return None
if self.isOptionMeta(section, option):
self.log.warning('set::option "%s.%s" cancelled, since it is a META option', (section, option))
return None
self.p.remove_option(section, option)
self.save()
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option) == 1
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryFloat(self.p.get(section, option))
def getDirectories(self, section, option):
value = self.p.get(section, option)
if value:
return map(str.strip, str.split(value, self.directories_delimiter))
return []
def getUnicode(self, section, option):
value = self.p.get(section, option).decode("unicode_escape")
return toUnicode(value).strip()
def getValues(self):
from couchpotato.environment import Env
values = {}
soft_chroot = Env.get("softchroot")
# TODO : There is two commented "continue" blocks (# COMMENTED_SKIPPING). They both are good...
# ... but, they omit output of values of hidden and non-readable options
# Currently, such behaviour could break the Web UI of CP...
# So, currently this two blocks are commented (but they are required to
# provide secure hidding of options.
for section in self.sections():
# COMMENTED_SKIPPING
# if not self.isSectionReadable(section):
# continue
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
# skip meta options:
if self.isOptionMeta(section, option_name):
continue
# COMMENTED_SKIPPING
# if not self.isOptionReadable(section, option_name):
# continue
value = self.get(option_name, section)
is_password = self.getType(section, option_name) == "password"
if is_password and value:
value = len(value) * "*"
# chrootify directory before sending to UI:
if (self.getType(section, option_name) == "directory") and value:
try:
value = soft_chroot.abs2chroot(value)
except:
value = ""
# chrootify directories before sending to UI:
if self.getType(section, option_name) == "directories":
if not value:
value = []
try:
value = map(soft_chroot.abs2chroot, value)
except:
value = []
values[section][option_name] = value
return values
def save(self):
with open(self.file, "wb") as configfile:
self.p.write(configfile)
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def getType(self, section, option):
tp = None
try:
tp = self.types[section][option]
except:
tp = "unicode" if not tp else tp
return tp
def addOptions(self, section_name, options):
# no additional actions (related to ro-rw options) are required here
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
"""Returns dict of UI-readable options
To check, whether the option is readable self.isOptionReadable() is used
"""
res = {}
# it is required to filter invisible options for UI, but also we should
# preserve original tree for server's purposes.
# So, next loops do one thing: copy options to res and in the process
# 1. omit NON-READABLE (for UI) options, and
# 2. put flags on READONLY options
for section_key in self.options.keys():
section_orig = self.options[section_key]
section_name = section_orig.get("name") if "name" in section_orig else section_key
if self.isSectionReadable(section_name):
section_copy = {}
section_copy_groups = []
for section_field in section_orig:
if section_field.lower() != "groups":
section_copy[section_field] = section_orig[section_field]
else:
for group_orig in section_orig["groups"]:
group_copy = {}
group_copy_options = []
for group_field in group_orig:
if group_field.lower() != "options":
group_copy[group_field] = group_orig[group_field]
else:
for option in group_orig[group_field]:
option_name = option.get("name")
# You should keep in mind, that READONLY = !IS_WRITABLE
# and IS_READABLE is a different thing
if self.isOptionReadable(section_name, option_name):
group_copy_options.append(option)
if not self.isOptionWritable(section_name, option_name):
option["readonly"] = True
if len(group_copy_options) > 0:
group_copy["options"] = group_copy_options
section_copy_groups.append(group_copy)
if len(section_copy_groups) > 0:
section_copy["groups"] = section_copy_groups
res[section_key] = section_copy
return res
def view(self, **kwargs):
return {"options": self.getOptions(), "values": self.getValues()}
def saveView(self, **kwargs):
section = kwargs.get("section")
option = kwargs.get("name")
value = kwargs.get("value")
if not self.isOptionWritable(section, option):
self.log.warning('Option "%s.%s" isn\'t writable', (section, option))
return {"success": False}
from couchpotato.environment import Env
soft_chroot = Env.get("softchroot")
if self.getType(section, option) == "directory":
value = soft_chroot.chroot2abs(value)
if self.getType(section, option) == "directories":
import json
value = json.loads(value)
if not (value and isinstance(value, list)):
value = []
value = map(soft_chroot.chroot2abs, value)
value = self.directories_delimiter.join(value)
# See if a value handler is attached, use that as value
new_value = fireEvent("setting.save.%s.%s" % (section, option), value, single=True)
self.set(section, option, (new_value if new_value else value).encode("unicode_escape"))
self.save()
# After save (for re-interval etc)
fireEvent("setting.save.%s.%s.after" % (section, option), single=True)
fireEvent("setting.save.%s.*.after" % section, single=True)
return {"success": True}
def isSectionReadable(self, section):
meta = "section_hidden" + self.optionMetaSuffix()
try:
return not self.p.getboolean(section, meta)
except:
pass
# by default - every section is readable:
return True
def isOptionReadable(self, section, option):
meta = option + self.optionMetaSuffix()
if self.p.has_option(section, meta):
meta_v = self.p.get(section, meta).lower()
return (meta_v == "rw") or (meta_v == "ro")
# by default - all is writable:
return True
def optionReadableCheckAndWarn(self, section, option):
x = self.isOptionReadable(section, option)
if not x:
self.log.warning('Option "%s.%s" isn\'t readable', (section, option))
return x
def isOptionWritable(self, section, option):
meta = option + self.optionMetaSuffix()
if self.p.has_option(section, meta):
return self.p.get(section, meta).lower() == "rw"
# by default - all is writable:
return True
def optionMetaSuffix(self):
return "_internal_meta"
def isOptionMeta(self, section, option):
""" A helper method for detecting internal-meta options in the ini-file
For a meta options used following names:
* section_hidden_internal_meta = (True | False) - for section visibility
* <OPTION>_internal_meta = (ro|rw|hidden) - for section visibility
"""
suffix = self.optionMetaSuffix()
return option.endswith(suffix)
def getProperty(self, identifier):
from couchpotato import get_db
db = get_db()
prop = None
try:
propert = db.get("property", identifier, with_doc=True)
prop = propert["doc"]["value"]
except ValueError:
propert = db.get("property", identifier)
fireEvent("database.delete_corrupted", propert.get("_id"))
except:
self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0)))
return prop
def setProperty(self, identifier, value=""):
from couchpotato import get_db
db = get_db()
try:
p = db.get("property", identifier, with_doc=True)
p["doc"].update({"identifier": identifier, "value": toUnicode(value)})
db.update(p["doc"])
except:
db.insert({"_t": "property", "identifier": identifier, "value": toUnicode(value)})
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
import re
import time
import traceback
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
class NzbIndex(NZBProvider, RSS):
urls = {
'download': 'http://www.nzbindex.nl/download/',
'api': 'http://www.nzbindex.nl/rss/',
}
http_time_between_calls = 1 # Seconds
def search(self, movie, quality):
results = []
if self.isDisabled():
from apscheduler.scheduler import Scheduler as Sched
from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
import logging
log = CPLog(__name__)
class Scheduler(Plugin):
crons = {}
intervals = {}
started = False
def __init__(self):
logging.getLogger('apscheduler').setLevel(logging.ERROR)
addEvent('schedule.cron', self.cron)
addEvent('schedule.interval', self.interval)
addEvent('schedule.start', self.start)
addEvent('schedule.restart', self.start)
addEvent('app.load', self.start)
self.sched = Sched(misfire_grace_time = 60)
def remove(self, identifier):
for type in ['interval', 'cron']:
try:
from couchpotato.core.helpers.variable import tryInt, tryFloat
from couchpotato.core.logger import CPLog
from datetime import timedelta
from hashlib import sha1
from multipartpost import MultipartPostHandler
import cookielib
import httplib
import json
import os
import re
import stat
import time
import urllib
import urllib2
log = CPLog(__name__)
class uTorrent(Downloader):
protocol = ['torrent', 'torrent_magnet']
utorrent_api = None
def connect(self):
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error(
'Config properties are not filled in correctly, port is missing.'
)
return False
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
pass
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
# Register environment settings
Env.set('encoding', encoding)
Env.set('uses_git', not options.nogit)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db'))
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
if not development:
atexit.register(cleanup)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Disable server access log
logging.getLogger('werkzeug').setLevel(logging.WARNING)
# Only run once when debugging
fire_load = False
if os.environ.get('WERKZEUG_RUN_MAIN') or not reloader:
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s' % options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s' % (category, message, filename, lineno))
warnings.showwarning = customwarn
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = base_path)
loader.run()
# Load migrations
initialize = True
db = Env.get('db_path')
if os.path.isfile(db.replace('sqlite:///', '')):
initialize = False
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
logging.getLogger('migrate').setLevel(logging.WARNING) # Disable logging for migration
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version = latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
if initialize:
fireEvent('app.initialize', in_order = True)
fire_load = True
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else ''
# Basic config
app.secret_key = api_key
config = {
'use_reloader': reloader,
'host': Env.setting('host', default = '0.0.0.0'),
'port': tryInt(Env.setting('port', default = 5000))
}
# Static path
app.static_folder = os.path.join(base_path, 'couchpotato', 'static')
web.add_url_rule('%s/static/<path:filename>' % api_key,
endpoint = 'static',
view_func = app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix = '%s/' % url_base)
app.register_blueprint(api, url_prefix = '%s/%s/' % (url_base, api_key))
# Some logging and fire load event
try: log.info('Starting server on port %(port)s' % config)
except: pass
if fire_load: fireEventAsync('app.load')
# Go go go!
try_restart = True
restart_tries = 5
while try_restart:
try:
app.run(**config)
except Exception, e:
try:
nr, msg = e
if nr == 48:
log.info('Already in use, try %s more time after few seconds' % restart_tries)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
class Loader(object):
do_restart = False
def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
self.options = getOptions(sys.argv[1:])
# Load settings
settings = Env.get('settings')
settings.setFile(self.options.config_file)
# Create data dir if needed
if self.options.data_dir:
self.data_dir = self.options.data_dir
else:
self.data_dir = os.path.expanduser(Env.setting('data_dir'))
if self.data_dir == '':
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, 'logs')
if not os.path.isdir(self.log_dir):
os.makedirs(self.log_dir)
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%H:%M:%S')
hdlr = handlers.RotatingFileHandler(
os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def addSignals(self):
signal.signal(signal.SIGINT, self.onExit)
signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(1))
from couchpotato.core.event import addEvent
addEvent('app.after_shutdown', self.afterShutdown)
def afterShutdown(self, restart):
self.do_restart = restart
def onExit(self, signal, frame):
from couchpotato.core.event import fireEvent
fireEvent('app.shutdown', single=True)
def run(self):
self.addSignals()
from couchpotato.runner import runCouchPotato
runCouchPotato(self.options,
base_path,
sys.argv[1:],
data_dir=self.data_dir,
log_dir=self.log_dir,
Env=Env)
if self.do_restart:
self.restart()
def restart(self):
try:
# remove old pidfile first
try:
if self.runAsDaemon():
try:
self.daemon.stop()
except:
pass
except:
self.log.critical(traceback.format_exc())
# Release log files and shutdown logger
logging.shutdown()
time.sleep(3)
args = [sys.executable] + [
os.path.join(base_path, os.path.basename(__file__))
] + sys.argv[1:]
subprocess.Popen(args)
except:
self.log.critical(traceback.format_exc())
def daemonize(self):
if self.runAsDaemon():
try:
from daemon import Daemon
self.daemon = Daemon(self.options.pid_file)
self.daemon.daemonize()
except SystemExit:
raise
except:
self.log.critical(traceback.format_exc())
def runAsDaemon(self):
return self.options.daemon and self.options.pid_file
from axl.axel import Event
from couchpotato.core.helpers.variable import mergeDicts, natcmp
from couchpotato.core.logger import CPLog
import threading
import traceback
log = CPLog(__name__)
events = {}
def runHandler(name, handler, *args, **kwargs):
try:
return handler(*args, **kwargs)
except:
from couchpotato.environment import Env
log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all()))
def addEvent(name, handler, priority = 100):
if events.get(name):
e = events[name]
else:
e = events[name] = Event(name = name, threads = 20, exc_info = True, traceback = True, lock = threading.RLock())
def createHandle(*args, **kwargs):
try:
parent = handler.im_self
bc = hasattr(parent, 'beforeCall')
if bc: parent.beforeCall(handler)
h = runHandler(name, handler, *args, **kwargs)
ac = hasattr(parent, 'afterCall')
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView('settings', self.view, docs = {
'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.',
'return': {'type': 'object', 'example': """{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}"""}
})
addApiView('settings.save', self.saveView, docs = {
'desc': 'Save setting to config file (settings.conf)',
'params': {
'section': {'desc': 'The section name in settings.conf'},
'name': {'desc': 'The option name'},
'value': {'desc': 'The value you want to save'},
}
})
addEvent('database.setup', self.databaseSetup)
self.file = None
self.p = None
self.log = None
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def databaseSetup(self):
fireEvent('database.setup_index', 'property', PropertyIndex)
def parser(self):
return self.p
def sections(self):
return self.p.sections()
def connectEvents(self):
addEvent('settings.options', self.addOptions)
addEvent('settings.register', self.registerDefaults)
addEvent('settings.save', self.save)
def registerDefaults(self, section_name, options = None, save = True):
if not options: options = {}
self.addSection(section_name)
for option_name, option in options.items():
self.setDefault(section_name, option_name, option.get('default', ''))
# Migrate old settings from old location to the new location
if option.get('migrate_from'):
if self.p.has_option(option.get('migrate_from'), option_name):
previous_value = self.p.get(option.get('migrate_from'), option_name)
self.p.set(section_name, option_name, previous_value)
self.p.remove_option(option.get('migrate_from'), option_name)
if option.get('type'):
self.setType(section_name, option_name, option.get('type'))
if save:
self.save()
def set(self, section, option, value):
return self.p.set(section, option, value)
def get(self, option = '', section = 'core', default = None, type = None):
try:
try: type = self.types[section][option]
except: type = 'unicode' if not type else type
if hasattr(self, 'get%s' % type.capitalize()):
return getattr(self, 'get%s' % type.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def delete(self, option = '', section = 'core'):
self.p.remove_option(section, option)
self.save()
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option) == 1
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryFloat(self.p.get(section, option))
def getUnicode(self, section, option):
value = self.p.get(section, option).decode('unicode_escape')
return toUnicode(value).strip()
def getValues(self):
values = {}
for section in self.sections():
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
is_password = False
try: is_password = self.types[section][option_name] == 'password'
except: pass
values[section][option_name] = self.get(option_name, section)
if is_password and values[section][option_name]:
values[section][option_name] = len(values[section][option_name]) * '*'
return values
def save(self):
with open(self.file, 'wb') as configfile:
self.p.write(configfile)
self.log.debug('Saved settings')
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def addOptions(self, section_name, options):
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
return self.options
def view(self, **kwargs):
return {
'options': self.getOptions(),
'values': self.getValues()
}
def saveView(self, **kwargs):
section = kwargs.get('section')
option = kwargs.get('name')
value = kwargs.get('value')
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True)
self.set(section, option, (new_value if new_value else value).encode('unicode_escape'))
self.save()
# After save (for re-interval etc)
fireEvent('setting.save.%s.%s.after' % (section, option), single = True)
fireEvent('setting.save.%s.*.after' % section, single = True)
return {
'success': True,
}
def getProperty(self, identifier):
from couchpotato import get_db
db = get_db()
prop = None
try:
propert = db.get('property', identifier, with_doc = True)
prop = propert['doc']['value']
except:
pass # self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0)))
return prop
def setProperty(self, identifier, value = ''):
from couchpotato import get_db
db = get_db()
try:
p = db.get('property', identifier, with_doc = True)
p['doc'].update({
'identifier': identifier,
'value': toUnicode(value),
})
db.update(p['doc'])
except:
db.insert({
'_t': 'property',
'identifier': identifier,
'value': toUnicode(value),
})
from couchpotato.core.helpers.encoding import isInt, ss
from couchpotato.core.logger import CPLog
from hashlib import sha1
from multipartpost import MultipartPostHandler
from datetime import timedelta
import os
import cookielib
import httplib
import json
import re
import time
import urllib
import urllib2
log = CPLog(__name__)
class uTorrent(Downloader):
type = ['torrent', 'torrent_magnet']
utorrent_api = None
def download(self, data, movie, filedata = None):
log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
from base64 import standard_b64encode
from couchpotato.core.downloaders.base import Downloader, StatusList
from couchpotato.core.helpers.encoding import ss
from couchpotato.core.helpers.variable import tryInt, md5
from couchpotato.core.logger import CPLog
from datetime import timedelta
import re
import shutil
import socket
import traceback
import xmlrpclib
log = CPLog(__name__)
class NZBGet(Downloader):
type = ['nzb']
url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc'
def download(self, data={}, movie={}, filedata=None):
if not filedata:
log.error('Unable to get NZB file: %s', traceback.format_exc())
return False
log.info('Sending "%s" to NZBGet.', data.get('name'))
url = self.url % {
'host': self.conf('host'),
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView('settings', self.view, docs = {
'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.',
'return': {'type': 'object', 'example': """{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}"""}
})
addApiView('settings.save', self.saveView, docs = {
'desc': 'Save setting to config file (settings.conf)',
'params': {
'section': {'desc': 'The section name in settings.conf'},
'option': {'desc': 'The option name'},
'value': {'desc': 'The value you want to save'},
}
})
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def parser(self):
return self.p
def sections(self):
return self.p.sections()
def connectEvents(self):
addEvent('settings.options', self.addOptions)
addEvent('settings.register', self.registerDefaults)
addEvent('settings.save', self.save)
def registerDefaults(self, section_name, options = None, save = True):
if not options: options = {}
self.addSection(section_name)
for option_name, option in options.items():
self.setDefault(section_name, option_name, option.get('default', ''))
# Migrate old settings from old location to the new location
if option.get('migrate_from'):
if self.p.has_option(option.get('migrate_from'), option_name):
previous_value = self.p.get(option.get('migrate_from'), option_name)
self.p.set(section_name, option_name, previous_value)
self.p.remove_option(option.get('migrate_from'), option_name)
if option.get('type'):
self.setType(section_name, option_name, option.get('type'))
if save:
self.save()
def set(self, section, option, value):
return self.p.set(section, option, value)
def get(self, option = '', section = 'core', default = None, type = None):
try:
try: type = self.types[section][option]
except: type = 'unicode' if not type else type
if hasattr(self, 'get%s' % type.capitalize()):
return getattr(self, 'get%s' % type.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def delete(self, option = '', section = 'core'):
self.p.remove_option(section, option)
self.save()
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option) == 1
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryFloat(self.p.get(section, option))
def getUnicode(self, section, option):
value = self.p.get(section, option).decode('unicode_escape')
return toUnicode(value).strip()
def getValues(self):
values = {}
for section in self.sections():
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
values[section][option_name] = self.get(option_name, section)
return values
def save(self):
with open(self.file, 'wb') as configfile:
self.p.write(configfile)
self.log.debug('Saved settings')
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def addOptions(self, section_name, options):
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
return self.options
def view(self, **kwargs):
return {
'options': self.getOptions(),
'values': self.getValues()
}
def saveView(self, **kwargs):
section = kwargs.get('section')
option = kwargs.get('name')
value = kwargs.get('value')
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True)
self.set(section, option, (new_value if new_value else value).encode('unicode_escape'))
self.save()
# After save (for re-interval etc)
fireEvent('setting.save.%s.%s.after' % (section, option), single = True)
fireEvent('setting.save.%s.*.after' % section, single = True)
return {
'success': True,
}
def getProperty(self, identifier):
from couchpotato import get_session
db = get_session()
prop = None
try:
propert = db.query(Properties).filter_by(identifier = identifier).first()
prop = propert.value
except:
pass
return prop
def setProperty(self, identifier, value = ''):
from couchpotato import get_session
try:
db = get_session()
p = db.query(Properties).filter_by(identifier = identifier).first()
if not p:
p = Properties()
db.add(p)
p.identifier = identifier
p.value = toUnicode(value)
db.commit()
except:
self.log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
os.remove(os.path.join(backup_path, backup_file))
else:
existing_backups.append((int(ints[0]), backup_file))
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('http_opener', requests.Session())
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value = api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application(
[],
log_function = lambda x: None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order = True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def test():
fireEvent('app.shutdown')
add_reload_hook(test)
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application, no_keep_alive = True, ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive = True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
# Do db stuff
db_path = os.path.join(data_dir, 'couchpotato.db')
# Backup before start and cleanup old databases
new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time())))
# Create path and copy
if not os.path.isdir(new_backup): os.makedirs(new_backup)
src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal']
for src_file in src_files:
if os.path.isfile(src_file):
shutil.copy2(src_file, os.path.join(new_backup, os.path.basename(src_file)))
# Remove older backups, keep backups 3 days or at least 3
backups = []
for directory in os.listdir(os.path.dirname(new_backup)):
backup = os.path.join(os.path.dirname(new_backup), directory)
if os.path.isdir(backup):
backups.append(backup)
total_backups = len(backups)
for backup in backups:
if total_backups > 3:
if int(os.path.basename(backup)) < time.time() - 259200:
for src_file in src_files:
b_file = os.path.join(backup, os.path.basename(src_file))
if os.path.isfile(b_file):
os.remove(b_file)
os.rmdir(backup)
total_backups -= 1
# Register environment settings
Env.set('encoding', encoding)
Env.set('app_dir', base_path)
Env.set('data_dir', data_dir)
Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log'))
Env.set('db_path', 'sqlite:///' + db_path)
Env.set('cache_dir', os.path.join(data_dir, 'cache'))
Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python')))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10)
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Check if database exists
db = Env.get('db_path')
db_exists = os.path.isfile(db_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = base_path)
loader.run()
# Load migrations
if db_exists:
from migrate.versioning.api import version_control, db_version, version, upgrade
repo = os.path.join(base_path, 'couchpotato', 'core', 'migration')
latest_db_version = version(repo)
try:
current_db_version = db_version(db, repo)
except:
version_control(db, repo, version = latest_db_version)
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not development:
log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
from couchpotato.core.settings.model import setup
setup()
# Fill database with needed stuff
if not db_exists:
fireEvent('app.initialize', in_order = True)
# Create app
from couchpotato import app
api_key = Env.setting('api_key')
url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else ''
# Basic config
app.secret_key = api_key
# app.debug = development
config = {
'use_reloader': reloader,
'host': Env.setting('host', default = '0.0.0.0'),
'port': tryInt(Env.setting('port', default = 5000))
}
# Static path
app.static_folder = os.path.join(base_path, 'couchpotato', 'static')
web.add_url_rule('api/%s/static/<path:filename>' % api_key,
endpoint = 'static',
view_func = app.send_static_file)
# Register modules
app.register_blueprint(web, url_prefix = '%s/' % url_base)
app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key))
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
# Go go go!
from tornado.ioloop import IOLoop
web_container = WSGIContainer(app)
web_container._log = _log
loop = IOLoop.instance()
application = Application([
(r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler),
(r'.*', FallbackHandler, dict(fallback = web_container)),
],
log_function = lambda x : None,
debug = config['use_reloader']
)
try_restart = True
restart_tries = 5
while try_restart:
try:
application.listen(config['port'], config['host'], no_keep_alive = True)
loop.start()
except Exception, e:
try:
nr, msg = e
if nr == 48:
log.info('Already in use, try %s more time after few seconds', restart_tries)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
class Transmission(Downloader):
protocol = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
trpc = None
def connect(self, reconnect = False):
# Load host from config and split out port.
host = cleanHost(self.conf('host'), protocol = False).split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
if not self.trpc or reconnect:
self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password'))
return self.trpc
def download(self, data = None, media = None, filedata = None):
if not media: media = {}
if not data: data = {}
log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol')))
if not self.connect():
return False
if not filedata and data.get('protocol') == 'torrent':
log.error('Failed sending torrent, no data')
return False
# Set parameters for adding torrent
params = {
'paused': self.conf('paused', default = False)
}
if self.conf('directory'):
if os.path.isdir(self.conf('directory')):
params['download-dir'] = self.conf('directory')
else:
log.error('Download directory from Transmission settings: %s doesn\'t exist', self.conf('directory'))
# Change parameters of torrent
torrent_params = {}
if data.get('seed_ratio'):
torrent_params['seedRatioLimit'] = tryFloat(data.get('seed_ratio'))
torrent_params['seedRatioMode'] = 1
if data.get('seed_time'):
torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time')) * 60
torrent_params['seedIdleMode'] = 1
# Send request to Transmission
if data.get('protocol') == 'torrent_magnet':
remote_torrent = self.trpc.add_torrent_uri(data.get('url'), arguments = params)
torrent_params['trackerAdd'] = self.torrent_trackers
else:
remote_torrent = self.trpc.add_torrent_file(b64encode(filedata), arguments = params)
if not remote_torrent:
log.error('Failed sending torrent to Transmission')
return False
# Change settings of added torrents
if torrent_params:
self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params)
log.info('Torrent sent to Transmission successfully.')
return self.downloadReturnId(remote_torrent['torrent-added']['hashString'])
def test(self):
if self.connect(True) and self.trpc.get_session():
return True
return False
def getAllDownloadStatus(self, ids):
log.debug('Checking Transmission download status.')
if not self.connect():
return []
release_downloads = ReleaseDownloadList(self)
return_params = {
'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit', 'files']
}
session = self.trpc.get_session()
queue = self.trpc.get_alltorrents(return_params)
if not (queue and queue.get('torrents')):
log.debug('Nothing in queue or error')
return []
for torrent in queue['torrents']:
if torrent['hashString'] in ids:
log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / isStalled=%s / eta=%s / uploadRatio=%s / isFinished=%s / incomplete-dir-enabled=%s / incomplete-dir=%s',
(torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent.get('isStalled', 'N/A'), torrent['eta'], torrent['uploadRatio'], torrent['isFinished'], session['incomplete-dir-enabled'], session['incomplete-dir']))
status = 'busy'
if torrent.get('isStalled') and not torrent['percentDone'] == 1 and self.conf('stalled_as_failed'):
status = 'failed'
elif torrent['status'] == 0 and torrent['percentDone'] == 1:
status = 'completed'
elif torrent['status'] in [5, 6]:
status = 'seeding'
if session['incomplete-dir-enabled'] and status == 'busy':
torrent_folder = session['incomplete-dir']
else:
torrent_folder = torrent['downloadDir']
torrent_files = []
for file_item in torrent['files']:
torrent_files.append(sp(os.path.join(torrent_folder, file_item['name'])))
release_downloads.append({
'id': torrent['hashString'],
'name': torrent['name'],
'status': status,
'original_status': torrent['status'],
'seed_ratio': torrent['uploadRatio'],
'timeleft': str(timedelta(seconds = torrent['eta'])),
'folder': sp(torrent_folder if len(torrent_files) == 1 else os.path.join(torrent_folder, torrent['name'])),
'files': '|'.join(torrent_files)
})
return release_downloads
def pause(self, release_download, pause = True):
if pause:
return self.trpc.stop_torrent(release_download['id'])
else:
return self.trpc.start_torrent(release_download['id'])
def removeFailed(self, release_download):
log.info('%s failed downloading, deleting...', release_download['name'])
return self.trpc.remove_torrent(release_download['id'], True)
def processComplete(self, release_download, delete_files = False):
log.debug('Requesting Transmission to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else ''))
return self.trpc.remove_torrent(release_download['id'], delete_files)
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView(
"settings",
self.view,
docs={
"desc": "Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.",
"return": {
"type": "object",
"example": """{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}""",
},
},
)
addApiView(
"settings.save",
self.saveView,
docs={
"desc": "Save setting to config file (settings.conf)",
"params": {
"section": {"desc": "The section name in settings.conf"},
"option": {"desc": "The option name"},
"value": {"desc": "The value you want to save"},
},
},
)
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def parser(self):
return self.p
def sections(self):
return self.p.sections()
def connectEvents(self):
addEvent("settings.options", self.addOptions)
addEvent("settings.register", self.registerDefaults)
addEvent("settings.save", self.save)
def registerDefaults(self, section_name, options={}, save=True):
self.addSection(section_name)
for option_name, option in options.iteritems():
self.setDefault(section_name, option_name, option.get("default", ""))
if option.get("type"):
self.setType(section_name, option_name, option.get("type"))
if save:
self.save(self)
def set(self, section, option, value):
return self.p.set(section, option, value)
def get(self, option="", section="core", default="", type=None):
try:
try:
type = self.types[section][option]
except:
type = "unicode" if not type else type
if hasattr(self, "get%s" % type.capitalize()):
return getattr(self, "get%s" % type.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option)
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryInt(self.p.get(section, option))
def getUnicode(self, section, option):
value = self.p.get(section, option).decode("unicode_escape")
return toUnicode(value).strip()
def getValues(self):
values = {}
for section in self.sections():
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
values[section][option_name] = self.get(option_name, section)
return values
def save(self):
with open(self.file, "wb") as configfile:
self.p.write(configfile)
self.log.debug("Saved settings")
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def addOptions(self, section_name, options):
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
return self.options
def view(self):
return jsonified({"options": self.getOptions(), "values": self.getValues()})
def saveView(self):
params = getParams()
section = params.get("section")
option = params.get("name")
value = params.get("value")
# See if a value handler is attached, use that as value
new_value = fireEvent("setting.save.%s.%s" % (section, option), value, single=True)
self.set(section, option, (new_value if new_value else value).encode("unicode_escape"))
self.save()
return jsonified({"success": True})
def getProperty(self, identifier):
from couchpotato import get_session
db = get_session()
try:
prop = db.query(Properties).filter_by(identifier=identifier).first()
return prop.value if prop else None
except:
return None
def setProperty(self, identifier, value=""):
from couchpotato import get_session
db = get_session()
p = db.query(Properties).filter_by(identifier=identifier).first()
if not p:
p = Properties()
db.add(p)
p.identifier = identifier
p.value = toUnicode(value)
db.commit()
class Settings():
options = {}
types = {}
def __init__(self):
addApiView('settings', self.view)
addApiView('settings.save', self.saveView)
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def parser(self):
return self.p
def sections(self):
return self.p.sections()
def connectEvents(self):
addEvent('settings.options', self.addOptions)
addEvent('settings.register', self.registerDefaults)
addEvent('settings.save', self.save)
def registerDefaults(self, section_name, options = {}, save = True):
self.addSection(section_name)
for option_name, option in options.iteritems():
self.setDefault(section_name, option_name, option.get('default', ''))
if option.get('type'):
self.setType(section_name, option_name, option.get('type'))
if save:
self.save(self)
def set(self, section, option, value):
return self.p.set(section, option, value)
def get(self, option = '', section = 'core', default = '', type = None):
try:
try: type = self.types[section][option]
except: type = 'unicode' if not type else type
if hasattr(self, 'get%s' % type.capitalize()):
return getattr(self, 'get%s' % type.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option)
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryInt(self.p.get(section, option))
def getUnicode(self, section, option):
value = self.p.get(section, option)
return toUnicode(value).strip()
def getValues(self):
values = {}
for section in self.sections():
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
values[section][option_name] = self.get(option_name, section)
return values
def save(self):
with open(self.file, 'wb') as configfile:
self.p.write(configfile)
self.log.debug('Saved settings')
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def addOptions(self, section_name, options):
if not self.options.get(section_name):
self.options[section_name] = options
else:
options['groups'] = self.options[section_name].get('groups') + options.get('groups')
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
return self.options
def view(self):
return jsonified({
'options': self.getOptions(),
'values': self.getValues()
})
def saveView(self):
params = getParams()
section = params.get('section')
option = params.get('name')
value = params.get('value')
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True)
self.set(section, option, new_value if new_value else value)
self.save()
return jsonified({
'success': True,
})
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import tryUrlencode, ss
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from urllib2 import URLError
from uuid import uuid4
import hashlib
import httplib
import json
import socket
import ssl
import sys
import traceback
import urllib2
log = CPLog(__name__)
class NZBVortex(Downloader):
type = ['nzb']
api_level = None
session_id = None
def download(self, data = {}, movie = {}, manual = False, filedata = None):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')) or not self.getApiLevel():
return
# Send the nzb
try:
nzb_filename = self.createFileName(data, filedata, movie)
from couchpotato.core.helpers.request import getParams, jsonified
from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
getImdb, link, symlink
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, File, Profile, Release, \
ReleaseInfo
from couchpotato.environment import Env
import errno
import os
import re
import shutil
import time
import traceback
log = CPLog(__name__)
class Renamer(Plugin):
renaming_started = False
checking_snatched = False
def __init__(self):
addApiView('renamer.scan', self.scanView, docs = {
'desc': 'For the renamer to check for new files to rename in a folder',
'params': {
'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'},
'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'},
'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'},
},
from base64 import b64encode
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.logger import CPLog
import httplib
import json
import os.path
import re
import urllib2
log = CPLog(__name__)
class Transmission(Downloader):
type = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data, movie, filedata = None):
log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
# Set parameters for Transmission
folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1]
folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep)
class Transmission(Downloader):
type = ['torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data, movie, manual=False, filedata=None):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return
log.debug('Sending "%s" (%s) to Transmission.',
(data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error(
'Config properties are not filled in correctly, port is missing.'
)
return False
# Set parameters for Transmission
folder_name = self.createFileName(data, filedata,
movie)[:-len(data.get('type')) - 1]
folder_path = os.path.join(self.conf('directory', default=''),
folder_name).rstrip(os.path.sep)
# Create the empty folder to download too
self.makeDir(folder_path)
params = {
'paused': self.conf('paused', default=0),
'download-dir': folder_path
}
torrent_params = {
'seedRatioLimit': self.conf('ratio'),
'seedRatioMode': (0 if self.conf('ratio') else 1)
}
if not filedata and data.get('type') == 'torrent':
log.error('Failed sending torrent, no data')
return False
# Send request to Transmission
try:
trpc = TransmissionRPC(host[0],
port=host[1],
username=self.conf('username'),
password=self.conf('password'))
if data.get('type') == 'torrent_magnet':
remote_torrent = trpc.add_torrent_uri(data.get('url'),
arguments=params)
torrent_params['trackerAdd'] = self.torrent_trackers
else:
remote_torrent = trpc.add_torrent_file(b64encode(filedata),
arguments=params)
# Change settings of added torrents
trpc.set_torrent(remote_torrent['torrent-added']['hashString'],
torrent_params)
return True
except Exception, err:
log.error('Failed to change settings for transfer: %s', err)
return False
from bencode import bencode, bdecode
from couchpotato.core.downloaders.base import Downloader, StatusList
from couchpotato.core.helpers.encoding import isInt, ss
from couchpotato.core.logger import CPLog
from datetime import timedelta
from hashlib import sha1
from multipartpost import MultipartPostHandler
import cookielib
import httplib
import json
import re
import time
import urllib
import urllib2
log = CPLog(__name__)
class uTorrent(Downloader):
type = ['torrent', 'torrent_magnet']
utorrent_api = None
def download(self, data, movie, filedata=None):
log.debug('Sending "%s" (%s) to uTorrent.',
(data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
class Settings(object):
options = {}
types = {}
def __init__(self):
addApiView('settings', self.view, docs = {
'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.',
'return': {'type': 'object', 'example': """{
// objects like in __init__.py of plugin
"options": {
"moovee" : {
"groups" : [{
"description" : "SD movies only",
"name" : "#alt.binaries.moovee",
"options" : [{
"default" : false,
"name" : "enabled",
"type" : "enabler"
}],
"tab" : "providers"
}],
"name" : "moovee"
}
},
// object structured like settings.conf
"values": {
"moovee": {
"enabled": false
}
}
}"""}
})
addApiView('settings.save', self.saveView, docs = {
'desc': 'Save setting to config file (settings.conf)',
'params': {
'section': {'desc': 'The section name in settings.conf'},
'option': {'desc': 'The option name'},
'value': {'desc': 'The value you want to save'},
}
})
def setFile(self, config_file):
self.file = config_file
self.p = ConfigParser.RawConfigParser()
self.p.read(config_file)
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
self.connectEvents()
def parser(self):
return self.p
def sections(self):
return self.p.sections()
def connectEvents(self):
addEvent('settings.options', self.addOptions)
addEvent('settings.register', self.registerDefaults)
addEvent('settings.save', self.save)
def registerDefaults(self, section_name, options = {}, save = True):
self.addSection(section_name)
for option_name, option in options.iteritems():
self.setDefault(section_name, option_name, option.get('default', ''))
if option.get('type'):
self.setType(section_name, option_name, option.get('type'))
if save:
self.save(self)
def set(self, section, option, value):
return self.p.set(section, option, value)
def get(self, option = '', section = 'core', default = None, type = None):
try:
try: type = self.types[section][option]
except: type = 'unicode' if not type else type
if hasattr(self, 'get%s' % type.capitalize()):
return getattr(self, 'get%s' % type.capitalize())(section, option)
else:
return self.getUnicode(section, option)
except:
return default
def getEnabler(self, section, option):
return self.getBool(section, option)
def getBool(self, section, option):
try:
return self.p.getboolean(section, option)
except:
return self.p.get(section, option) == 1
def getInt(self, section, option):
try:
return self.p.getint(section, option)
except:
return tryInt(self.p.get(section, option))
def getFloat(self, section, option):
try:
return self.p.getfloat(section, option)
except:
return tryInt(self.p.get(section, option))
def getUnicode(self, section, option):
value = self.p.get(section, option).decode('unicode_escape')
return toUnicode(value).strip()
def getValues(self):
values = {}
for section in self.sections():
values[section] = {}
for option in self.p.items(section):
(option_name, option_value) = option
values[section][option_name] = self.get(option_name, section)
return values
def save(self):
with open(self.file, 'wb') as configfile:
self.p.write(configfile)
self.log.debug('Saved settings')
def addSection(self, section):
if not self.p.has_section(section):
self.p.add_section(section)
def setDefault(self, section, option, value):
if not self.p.has_option(section, option):
self.p.set(section, option, value)
def setType(self, section, option, type):
if not self.types.get(section):
self.types[section] = {}
self.types[section][option] = type
def addOptions(self, section_name, options):
if not self.options.get(section_name):
self.options[section_name] = options
else:
self.options[section_name] = mergeDicts(self.options[section_name], options)
def getOptions(self):
return self.options
def view(self):
return {
'options': self.getOptions(),
'values': self.getValues()
}
def saveView(self, **kwargs):
section = kwargs.get('section')
option = kwargs.get('name')
value = kwargs.get('value')
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True)
self.set(section, option, (new_value if new_value else value).encode('unicode_escape'))
self.save()
# After save (for re-interval etc)
fireEvent('setting.save.%s.%s.after' % (section, option), single = True)
return {
'success': True,
}
def getProperty(self, identifier):
from couchpotato import get_session
db = get_session()
prop = None
try:
propert = db.query(Properties).filter_by(identifier = identifier).first()
prop = propert.value
except:
pass
return prop
def setProperty(self, identifier, value = ''):
from couchpotato import get_session
db = get_session()
p = db.query(Properties).filter_by(identifier = identifier).first()
if not p:
p = Properties()
db.add(p)
p.identifier = identifier
p.value = toUnicode(value)
db.commit()
from apscheduler.scheduler import Scheduler as Sched
from couchpotato.core.event import addEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
import logging
log = CPLog(__name__)
class Scheduler(Plugin):
crons = {}
intervals = {}
started = False
def __init__(self):
addEvent("schedule.cron", self.cron)
addEvent("schedule.interval", self.interval)
addEvent("schedule.start", self.start)
addEvent("schedule.restart", self.start)
addEvent("app.load", self.start)
self.sched = Sched(misfire_grace_time=60)
def remove(self, identifier):
for type in ["interval", "cron"]:
try:
self.sched.unschedule_job(getattr(self, type)[identifier]["job"])
log.debug("%s unscheduled %s", (type.capitalize(), identifier))
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss
from couchpotato.core.helpers.variable import getExt, getImdb, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import File, Movie
from enzyme.exceptions import NoParserError, ParseError
from guessit import guess_movie_info
from subliminal.videos import Video
import enzyme
import os
import re
import time
import traceback
log = CPLog(__name__)
class Scanner(Plugin):
minimal_filesize = {
'media': 314572800, # 300MB
'trailer': 1048576, # 1MB
}
ignored_in_path = [
'extracting', '_unpack', '_failed_', '_unknown_', '_exists_',
'_failed_remove_', '_failed_rename_', '.appledouble', '.appledb',
'.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo'
] #unpacking, smb-crap, hidden files
ignore_names = [
'extract', 'extracting', 'extracted', 'movie', 'movies', 'film',
from base64 import standard_b64encode
from couchpotato.core.downloaders.base import Downloader, StatusList
from couchpotato.core.helpers.encoding import ss
from couchpotato.core.helpers.variable import tryInt, md5
from couchpotato.core.logger import CPLog
from datetime import timedelta
import re
import shutil
import socket
import traceback
import xmlrpclib
log = CPLog(__name__)
class NZBGet(Downloader):
type = ['nzb']
url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc'
def download(self, data = {}, movie = {}, filedata = None):
if not filedata:
log.error('Unable to get NZB file: %s', traceback.format_exc())
return False
log.info('Sending "%s" to NZBGet.', data.get('name'))
url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')}
nzb_name = ss('%s.nzb' % self.createNzbName(data, movie))
#!/usr/bin/env python
from os.path import dirname
from signal import signal, SIGTERM
import os
import subprocess
import sys
# Root path
base_path = dirname(os.path.abspath(__file__))
# Insert local directories into path
sys.path.insert(0, os.path.join(base_path, 'libs'))
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
# Get options via arg
from couchpotato.runner import getOptions
from couchpotato.core.helpers.variable import getDataDir
options = getOptions(base_path, sys.argv[1:])
data_dir = getDataDir()
def start():
try:
args = [sys.executable] + [os.path.join(base_path, __file__)] + sys.argv[1:]
new_environ = os.environ.copy()
new_environ['cp_main'] = 'true'
if os.name == 'nt':
for key, value in new_environ.iteritems():
def runCouchPotato(options, base_path, args, data_dir=None, log_dir=None, Env=None, desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ("ANSI_X3.4-1968", "US-ASCII", "ASCII"):
encoding = "UTF-8"
Env.set("encoding", encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, "database"))
old_db_path = os.path.join(data_dir, "couchpotato.db")
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, "db_backup"))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path):
os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall("\d+", backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, "%s.tar.gz" % int(time.time())))
zipf = tarfile.open(new_backup, "w:gz")
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(
os.path.join(root, zfilename),
arcname="database/%s" % os.path.join(root[len(db_path) + 1 :], zfilename),
)
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, "cache"))
python_cache = sp(os.path.join(cache_dir, "python"))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
session = requests.Session()
session.max_redirects = 5
# Register environment settings
Env.set("app_dir", sp(base_path))
Env.set("data_dir", sp(data_dir))
Env.set("log_path", sp(os.path.join(log_dir, "CouchPotato.log")))
Env.set("db", db)
Env.set("http_opener", session)
Env.set("cache_dir", cache_dir)
Env.set("cache", FileSystemCache(python_cache))
Env.set("console_log", options.console_log)
Env.set("quiet", options.quiet)
Env.set("desktop", desktop)
Env.set("daemonized", options.daemon)
Env.set("args", args)
Env.set("options", options)
# Determine debug
debug = options.debug or Env.setting("debug", default=False, type="bool")
Env.set("debug", debug)
# Development
development = Env.setting("development", default=False, type="bool")
Env.set("dev", development)
# Disable logging for some modules
for logger_name in ["enzyme", "guessit", "subliminal", "apscheduler", "tornado", "requests"]:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ["gntp"]:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get("desktop") and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%m-%d %H:%M:%S")
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, "INFO")
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get("log_path"), "a", 500000, 10, encoding=Env.get("encoding"))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug("Started with options %s", options)
# Check soft-chroot dir exists:
try:
# Load Soft-Chroot
soft_chroot = Env.get("softchroot")
soft_chroot_dir = Env.setting("soft_chroot", section="core", default=None, type="unicode")
soft_chroot.initialize(soft_chroot_dir)
except SoftChrootInitError as exc:
log.error(exc)
return
except:
log.error("Unable to check whether SOFT-CHROOT is defined")
return
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error(
"Shutting down as CP needs some space to work. You'll get corrupted data otherwise. Only %sMB left",
available_space,
)
return
except:
log.error("Failed getting diskspace: %s", traceback.format_exc())
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning("%s %s %s line:%s", (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ("/" + Env.setting("url_base").lstrip("/") + "/") if Env.setting("url_base") else "/"
Env.set("web_base", web_base)
api_key = Env.setting("api_key")
if not api_key:
api_key = uuid4().hex
Env.setting("api_key", value=api_key)
api_base = r"%sapi/%s/" % (web_base, api_key)
Env.set("api_base", api_base)
# Basic config
host = Env.setting("host", default="0.0.0.0")
host6 = Env.setting("host6", default="::")
config = {
"use_reloader": reloader,
"port": tryInt(Env.setting("port", default=5050)),
"host": host if host and len(host) > 0 else "0.0.0.0",
"host6": host6 if host6 and len(host6) > 0 else "::",
"ssl_cert": Env.setting("ssl_cert", default=None),
"ssl_key": Env.setting("ssl_key", default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config["use_reloader"],
gzip=True,
cookie_secret=api_key,
login_url="%slogin/" % web_base,
)
Env.set("app", application)
# Request handlers
application.add_handlers(
".*$",
[
(r"%snonblock/(.*)(/?)" % api_base, NonBlockHandler),
# API handlers
(r"%s(.*)(/?)" % api_base, ApiHandler), # Main API handler
(r"%sgetkey(/?)" % web_base, KeyHandler), # Get API key
(r"%s" % api_base, RedirectHandler, {"url": web_base + "docs/"}), # API docs
# Login handlers
(r"%slogin(/?)" % web_base, LoginHandler),
(r"%slogout(/?)" % web_base, LogoutHandler),
# Catch all webhandlers
(r"%s(.*)(/?)" % web_base, WebHandler),
(r"(.*)", WebHandler),
],
)
# Static paths
static_path = "%sstatic/" % web_base
for dir_name in ["fonts", "images", "scripts", "style"]:
application.add_handlers(
".*$",
[
(
"%s%s/(.*)" % (static_path, dir_name),
StaticFileHandler,
{"path": sp(os.path.join(base_path, "couchpotato", "static", dir_name))},
)
],
)
Env.set("static_path", static_path)
# Load configs & plugins
loader = Env.get("loader")
loader.preload(root=sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent("database.setup")
if not db_exists:
fireEvent("app.initialize", in_order=True)
fireEvent("app.migrate")
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent("app.shutdown")
add_reload_hook(reload_hook)
# Some logging and fire load event
try:
log.info("Starting server on port %(port)s", config)
except:
pass
fireEventAsync("app.load")
ssl_options = None
if config["ssl_cert"] and config["ssl_key"]:
ssl_options = {"certfile": config["ssl_cert"], "keyfile": config["ssl_key"]}
server = HTTPServer(application, no_keep_alive=True, ssl_options=ssl_options)
try_restart = True
restart_tries = 5
while try_restart:
try:
if config["host"].startswith("unix:"):
server.add_socket(bind_unix_socket(config["host"][5:]))
else:
server.listen(config["port"], config["host"])
if Env.setting("ipv6", default=False):
try:
server.listen(config["port"], config["host6"])
except:
log.info2("Tried to bind to IPV6 but failed")
loop.start()
server.close_all_connections()
server.stop()
loop.close(all_fds=True)
except Exception as e:
log.error("Failed starting: %s", traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
"Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds",
(config.get("port"), restart_tries),
)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except ValueError:
return
except:
pass
raise
try_restart = False
from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import isInt
from couchpotato.core.logger import CPLog
import httplib
import json
import urllib
import urllib2
log = CPLog(__name__)
class Synology(Downloader):
type = ['torrent_magnet']
log = CPLog(__name__)
def download(self, data, movie, manual = False, filedata = None):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return
log.error('Sending "%s" (%s) to Synology.', (data.get('name'), data.get('type')))
# Load host from config and split out port.
host = self.conf('host').split(':')
if not isInt(host[1]):
log.error('Config properties are not filled in correctly, port is missing.')
return False
if data.get('type') == 'torrent':
log.error('Can\'t add binary torrent file')
from axl.axel import Event
from couchpotato.core.helpers.variable import mergeDicts, natcmp
from couchpotato.core.logger import CPLog
import threading
import traceback
log = CPLog(__name__)
events = {}
def runHandler(name, handler, *args, **kwargs):
try:
return handler(*args, **kwargs)
except:
from couchpotato.environment import Env
log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all()))
def addEvent(name, handler, priority = 100):
if events.get(name):
e = events[name]
else:
e = events[name] = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock())
def createHandle(*args, **kwargs):
try:
parent = handler.im_self
bc = hasattr(parent, 'beforeCall')
if bc: parent.beforeCall(handler)
h = runHandler(name, handler, *args, **kwargs)
ac = hasattr(parent, 'afterCall')
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
old_db_path = os.path.join(data_dir, 'couchpotato.db')
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try: os.remove(os.path.join(root, backup_file))
except: pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
session = requests.Session()
session.max_redirects = 5
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('http_opener', session)
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
# Check soft-chroot dir exists:
try:
# Load Soft-Chroot
soft_chroot = Env.get('softchroot')
soft_chroot_dir = Env.setting('soft_chroot', section = 'core', default = None, type='unicode' )
soft_chroot.initialize(soft_chroot_dir)
except SoftChrootInitError as exc:
log.error(exc)
return
except:
log.error('Unable to check whether SOFT-CHROOT is defined')
return
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error('Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space)
return
except:
log.error('Failed getting diskspace: %s', traceback.format_exc())
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value = api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
host6 = Env.setting('host6', default = '::')
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'host6': host6 if host6 and len(host6) > 0 else '::',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application(
[],
log_function = lambda x: None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order = True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent('app.shutdown')
add_reload_hook(reload_hook)
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
ssl_options = None
if config['ssl_cert'] and config['ssl_key']:
ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
}
server = HTTPServer(application, no_keep_alive = True, ssl_options = ssl_options)
run_tries = 5
assert run_tries > 0
while run_tries > 0:
run_tries -= 1
try:
server.listen(config['port'], config['host'])
if Env.setting('ipv6', default = False):
try: server.listen(config['port'], config['host6'])
except: log.info2('Tried to bind to IPV6 but failed')
loop.start()
# on shutting down without exception
server.close_all_connections()
server.stop()
loop.close(all_fds = True)
except SocketError as e:
# here we will handle just two errors :
# errno.EADDRINUSE = 98 - address in use
# errno.ELNRNG = 48 - port in use (strange, but I took it from old code)
# TODO : check, that value 48 still actual
if e.errno in [ errno.EADDRINUSE, errno.ELNRNG ]:
if run_tries > 0:
log.warning('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds',
(config.get('port'), run_tries))
time.sleep(2)
continue
# not ugly error message
log.error('Failed starting: Port (%s) needed for CouchPotato is already in use',
(config['port']))
# we will not raise this exception again, because no additional actions are required:
return
log.error('Failed starting: %s', traceback.format_exc())
raise
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
raise
pass # while run_tries>0
