def __init__(self): # Get options via arg from couchpotato.runner import getOptions self.options = getOptions(base_path, sys.argv[1:]) # Load settings settings = Env.get('settings') settings.setFile(self.options.config_file) # Create data dir if needed self.data_dir = os.path.expanduser(Env.setting('data_dir')) if self.data_dir == '': self.data_dir = getDataDir() if not os.path.isdir(self.data_dir): os.makedirs(self.data_dir) # Create logging dir self.log_dir = os.path.join(self.data_dir, 'logs'); if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir) # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr)
def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents()
def __init__(self): # Get options via arg from couchpotato.runner import getOptions from couchpotato.core.helpers.variable import getDataDir self.options = getOptions(base_path, sys.argv[1:]) self.data_dir = getDataDir() # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) if self.options.daemon: formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') hdlr = handlers.RotatingFileHandler(os.path.join(self.data_dir, 'logs', 'error.log'), 'a', 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr)
def __init__(self): # Get options via arg from couchpotato.runner import getOptions self.options = getOptions(sys.argv[1:]) # Load settings settings = Env.get("settings") settings.setFile(self.options.config_file) # Create data dir if needed if self.options.data_dir: self.data_dir = self.options.data_dir else: self.data_dir = os.path.expanduser(Env.setting("data_dir")) if self.data_dir == "": self.data_dir = getDataDir() if not os.path.isdir(self.data_dir): os.makedirs(self.data_dir) # Create logging dir self.log_dir = os.path.join(self.data_dir, "logs") if not os.path.isdir(self.log_dir): os.makedirs(self.log_dir) # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%H:%M:%S") hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, "error.log"), "a", 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr)
from bencode import bencode, bdecode from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.logger import CPLog from hashlib import sha1 from multipartpost import MultipartPostHandler import cookielib import httplib import re import time import urllib import urllib2 log = CPLog(__name__) class uTorrent(Downloader): type = ["torrent", "torrent_magnet"] utorrent_api = None def download(self, data, movie, filedata=None): log.debug('Sending "%s" (%s) to uTorrent.', (data.get("name"), data.get("type"))) # Load host from config and split out port. host = self.conf("host").split(":") if not isInt(host[1]): log.error("Config properties are not filled in correctly, port is missing.") return False
from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog import json import requests log = CPLog(__name__) class Synology(Downloader): type = ['nzb', 'torrent', 'torrent_magnet'] log = CPLog(__name__) def download(self, data, movie, filedata = None): response = False log.error('Sending "%s" (%s) to Synology.', (data['name'], data['type'])) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False try: # Send request to Synology srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password')) if data['type'] == 'torrent_magnet': log.info('Adding torrent URL %s', data['url']) response = srpc.create_task(url = data['url'])
getImdb, link, symlink, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ ReleaseInfo from couchpotato.environment import Env from unrar2 import RarFile import errno import fnmatch import os import re import shutil import time import traceback log = CPLog(__name__) class Renamer(Plugin): renaming_started = False checking_snatched = False def __init__(self): addApiView('renamer.scan', self.scanView, docs = { 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { 'async': {'desc': 'Optional: Set to 1 if you dont want to fire the renamer.scan asynchronous.'}, 'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'}, 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, },
from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss from couchpotato.core.helpers.request import jsonified from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release from couchpotato.environment import Env import errno import os import re import shutil import traceback log = CPLog(__name__) class Renamer(Plugin): renaming_started = False def __init__(self): addApiView('renamer.scan', self.scanView, docs = { 'desc': 'For the renamer to check for new files to rename', }) addEvent('renamer.scan', self.scan) addEvent('renamer.check_snatched', self.checkSnatched)
from base64 import b64encode from couchpotato.core.downloaders.base import Downloader, ReleaseDownloadList from couchpotato.core.helpers.encoding import isInt, sp from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from datetime import timedelta import httplib import json import os.path import re import urllib2 log = CPLog(__name__) class Transmission(Downloader): protocol = ['torrent', 'torrent_magnet'] log = CPLog(__name__) trpc = None def connect(self): # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False if not self.trpc: self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url'), username = self.conf('username'), password = self.conf('password'))
class YGG(TorrentProvider, MovieProvider): """ Couchpotato plugin to search movies torrents on www.yggtorrent.com. .. seealso:: YarrProvider.login, Plugin.wait """ url_scheme = 'https' domain_name = 'yggtorrent.is' limit = 50 http_time_between_calls = 0 log = CPLog(__name__) def __init__(self): """ Default constructor """ TorrentProvider.__init__(self) MovieProvider.__init__(self) path_www = YGG.url_scheme + '://' + YGG.domain_name self.urls = { 'login': path_www + '/user/login', 'login_check': path_www + '/user/account', 'search': path_www + '/engine/search?{0}', 'torrent': path_www + '/torrent', 'url': path_www + '/engine/download_torrent?id={0}' } def getLoginParams(self): """ Return YGG login parameters. .. seealso:: YarrProvider.getLoginParams """ return { 'id': self.conf('username'), 'pass': self.conf('password') } def loginSuccess(self, output): """ Check server's response on authentication. .. seealso:: YarrProvider.loginSuccess """ return len(output) == 0 def loginCheckSuccess(self, output): """ Check if we are still connected. .. seealso:: YarrProvider.loginCheckSuccess """ result = False soup = BeautifulSoup(output, 'html.parser') if soup.find(text=u'Déconnexion'): result = True return result def getMoreInfo(self, nzb): """ Get details about a torrent. .. seealso:: MovieSearcher.correctRelease """ data = self.getHTMLData(nzb['detail_url']) soup = BeautifulSoup(data, 'html.parser') description = soup.find(class_='description-header').find_next('div') if description: nzb['description'] = description.prettify() line = soup.find(text=u'Uploadé le').find_next('td') added = datetime.strptime(line.getText().split('(')[0].strip(), '%d/%m/%Y %H:%M') nzb['age'] = (datetime.now() - added).days self.log.debug(nzb['age']) def extraCheck(self, nzb): """ Exclusion when movie's description contains more than one IMDB reference to prevent a movie bundle downloading. CouchPotato is not able to extract a specific movie from an archive. .. seealso:: MovieSearcher.correctRelease """ result = True ids = getImdb(nzb.get('description', ''), multiple=True) if len(ids) not in [0, 1]: YGG.log.info('Too much IMDB ids: {0}'.format(', '.join(ids))) result = False return result def parseText(self, node): """ Retrieve the text content from a HTML node. """ return node.getText().strip() def _searchOnTitle(self, title, media, quality, results, offset=0): """ Do a search based on possible titles. This function doesn't check the quality because CouchPotato do the job when parsing results. Furthermore the URL must stay generic to use native CouchPotato caching feature. .. seealso:: YarrProvider.search """ try: params = { 'category': 2145, # Film/Vidéo 'description': '', 'do': 'search', 'file': '', 'name': simplifyString(title), 'sub_category': 'all', 'uploader': '' } if offset > 0: params['page'] = offset * YGG.limit url = self.urls['search'].format(tryUrlencode(params)) data = self.getHTMLData(url) soup = BeautifulSoup(data, 'html.parser') filter_ = '^{0}'.format(self.urls['torrent']) for link in soup.find_all(href=re.compile(filter_)): detail_url = link['href'] if re.search(u'/filmvidéo/(film|animation|documentaire)/', detail_url): name = self.parseText(link) id_ = tryInt(re.search('/(\d+)-[^/\s]+$', link['href']). group(1)) columns = link.parent.parent.find_all('td') size = self.parseSize(self.parseText(columns[5])) seeders = tryInt(self.parseText(columns[7])) leechers = tryInt(self.parseText(columns[8])) result = { 'id': id_, 'name': name, 'seeders': seeders, 'leechers': leechers, 'size': size, 'url': self.urls['url'].format(id_), 'detail_url': detail_url, 'verified': True, 'get_more_info': self.getMoreInfo, 'extra_check': self.extraCheck } results.append(result) YGG.log.debug(result) # Get next page if we don't have all results pagination = soup.find('ul', class_='pagination') if pagination: for page in pagination.find_all('li'): next_ = tryInt(self.parseText(page.find('a'))) if next_ > offset + 1: self._searchOnTitle(title, media, quality, results, offset + 1) break except: YGG.log.error('Failed searching release from {0}: {1}'. format(self.getName(), traceback.format_exc()))
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = sp(os.path.join(data_dir, 'database')) # Check if database exists db = SuperThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups backup_path = sp(os.path.join(data_dir, 'db_backup')) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): for backup_file in sorted(files): ints = re.findall('\d+', backup_file) # Delete non zip files if len(ints) != 1: os.remove(os.path.join(backup_path, backup_file)) else: existing_backups.append((int(ints[0]), backup_file)) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: os.remove(os.path.join(backup_path, eb[1])) # Create new backup new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') for root, dirs, files in os.walk(db_path): for zfilename in files: zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename)) zipf.close() # Open last db.open() else: db.create() # Force creation of cachedir log_dir = sp(log_dir) cache_dir = sp(os.path.join(data_dir, 'cache')) python_cache = sp(os.path.join(cache_dir, 'python')) if not os.path.exists(cache_dir): os.mkdir(cache_dir) if not os.path.exists(python_cache): os.mkdir(python_cache) # Register environment settings Env.set('app_dir', sp(base_path)) Env.set('data_dir', sp(data_dir)) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) Env.set('cache_dir', cache_dir) Env.set('cache', FileSystemCache(python_cache)) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding')) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors # noinspection PyUnresolvedReferences import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') if not api_key: api_key = uuid4().hex Env.setting('api_key', value = api_key) api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default = '0.0.0.0') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } # Load the app application = Application( [], log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, login_url = '%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers(".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = sp(base_path)) loader.run() # Fill database with needed stuff fireEvent('database.setup') if not db_exists: fireEvent('app.initialize', in_order = True) fireEvent('app.migrate') # Go go go! from tornado.ioloop import IOLoop from tornado.autoreload import add_reload_hook loop = IOLoop.current() # Reload hook def test(): fireEvent('app.shutdown') add_reload_hook(test) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { 'certfile': config['ssl_cert'], 'keyfile': config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) try_restart = True restart_tries = 5 while try_restart: try: server.listen(config['port'], config['host']) loop.start() except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e if nr == 48: log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries)) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
class Settings(object): options = {} types = {} def __init__(self): addApiView( 'settings', self.view, docs={ 'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.', 'return': { 'type': 'object', 'example': """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }""" } }) addApiView('settings.save', self.saveView, docs={ 'desc': 'Save setting to config file (settings.conf)', 'params': { 'section': { 'desc': 'The section name in settings.conf' }, 'name': { 'desc': 'The option name' }, 'value': { 'desc': 'The value you want to save' }, } }) addEvent('database.setup', self.databaseSetup) self.file = None self.p = None self.log = None self.directories_delimiter = "::" def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def databaseSetup(self): fireEvent('database.setup_index', 'property', PropertyIndex) def parser(self): return self.p def sections(self): res = filter(self.isSectionReadable, self.p.sections()) return res def connectEvents(self): addEvent('settings.options', self.addOptions) addEvent('settings.register', self.registerDefaults) addEvent('settings.save', self.save) def registerDefaults(self, section_name, options=None, save=True): if not options: options = {} self.addSection(section_name) for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get('default', '')) # Set UI-meta for option (hidden/ro/rw) if option.get('ui-meta'): value = option.get('ui-meta') if value: value = value.lower() if value in ['hidden', 'rw', 'ro']: meta_option_name = option_name + self.optionMetaSuffix( ) self.setDefault(section_name, meta_option_name, value) else: self.log.warning( 'Wrong value for option %s.%s : ui-meta can not be equal to "%s"', (section_name, option_name, value)) # Migrate old settings from old location to the new location if option.get('migrate_from'): if self.p.has_option(option.get('migrate_from'), option_name): previous_value = self.p.get(option.get('migrate_from'), option_name) self.p.set(section_name, option_name, previous_value) self.p.remove_option(option.get('migrate_from'), option_name) if option.get('type'): self.setType(section_name, option_name, option.get('type')) if save: self.save() def set(self, section, option, value): if not self.isOptionWritable(section, option): self.log.warning('set::option "%s.%s" isn\'t writable', (section, option)) return None if self.isOptionMeta(section, option): self.log.warning( 'set::option "%s.%s" cancelled, since it is a META option', (section, option)) return None return self.p.set(section, option, value) def get(self, option='', section='core', default=None, type=None): if self.isOptionMeta(section, option): self.log.warning( 'get::option "%s.%s" cancelled, since it is a META option', (section, option)) return None tp = type try: tp = self.getType(section, option) if not tp else tp if hasattr(self, 'get%s' % tp.capitalize()): return getattr(self, 'get%s' % tp.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def delete(self, option='', section='core'): if not self.isOptionWritable(section, option): self.log.warning('delete::option "%s.%s" isn\'t writable', (section, option)) return None if self.isOptionMeta(section, option): self.log.warning( 'set::option "%s.%s" cancelled, since it is a META option', (section, option)) return None self.p.remove_option(section, option) self.save() def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) == 1 def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryFloat(self.p.get(section, option)) def getDirectories(self, section, option): value = self.p.get(section, option) if value: return map(str.strip, str.split(value, self.directories_delimiter)) return [] def getUnicode(self, section, option): value = self.p.get(section, option).decode('unicode_escape') return toUnicode(value).strip() def getValues(self): from couchpotato.environment import Env values = {} soft_chroot = Env.get('softchroot') # TODO : There is two commented "continue" blocks (# COMMENTED_SKIPPING). They both are good... # ... but, they omit output of values of hidden and non-readable options # Currently, such behaviour could break the Web UI of CP... # So, currently this two blocks are commented (but they are required to # provide secure hidding of options. for section in self.sections(): # COMMENTED_SKIPPING #if not self.isSectionReadable(section): # continue values[section] = {} for option in self.p.items(section): (option_name, option_value) = option #skip meta options: if self.isOptionMeta(section, option_name): continue # COMMENTED_SKIPPING #if not self.isOptionReadable(section, option_name): # continue value = self.get(option_name, section) is_password = self.getType(section, option_name) == 'password' if is_password and value: value = len(value) * '*' # chrootify directory before sending to UI: if (self.getType(section, option_name) == 'directory') and value: try: value = soft_chroot.abs2chroot(value) except: value = "" # chrootify directories before sending to UI: if (self.getType(section, option_name) == 'directories'): if (not value): value = [] try: value = map(soft_chroot.abs2chroot, value) except: value = [] values[section][option_name] = value return values def save(self): with open(self.file, 'wb') as configfile: self.p.write(configfile) def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def getType(self, section, option): tp = None try: tp = self.types[section][option] except: tp = 'unicode' if not tp else tp return tp def addOptions(self, section_name, options): # no additional actions (related to ro-rw options) are required here if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): """Returns dict of UI-readable options To check, whether the option is readable self.isOptionReadable() is used """ res = {} # it is required to filter invisible options for UI, but also we should # preserve original tree for server's purposes. # So, next loops do one thing: copy options to res and in the process # 1. omit NON-READABLE (for UI) options, and # 2. put flags on READONLY options for section_key in self.options.keys(): section_orig = self.options[section_key] section_name = section_orig.get( 'name') if 'name' in section_orig else section_key if self.isSectionReadable(section_name): section_copy = {} section_copy_groups = [] for section_field in section_orig: if section_field.lower() != 'groups': section_copy[section_field] = section_orig[ section_field] else: for group_orig in section_orig['groups']: group_copy = {} group_copy_options = [] for group_field in group_orig: if group_field.lower() != 'options': group_copy[group_field] = group_orig[ group_field] else: for option in group_orig[group_field]: option_name = option.get('name') # You should keep in mind, that READONLY = !IS_WRITABLE # and IS_READABLE is a different thing if self.isOptionReadable( section_name, option_name): group_copy_options.append(option) if not self.isOptionWritable( section_name, option_name): option['readonly'] = True if len(group_copy_options) > 0: group_copy['options'] = group_copy_options section_copy_groups.append(group_copy) if len(section_copy_groups) > 0: section_copy['groups'] = section_copy_groups res[section_key] = section_copy return res def view(self, **kwargs): return {'options': self.getOptions(), 'values': self.getValues()} def saveView(self, **kwargs): section = kwargs.get('section') option = kwargs.get('name') value = kwargs.get('value') if not self.isOptionWritable(section, option): self.log.warning('Option "%s.%s" isn\'t writable', (section, option)) return { 'success': False, } from couchpotato.environment import Env soft_chroot = Env.get('softchroot') if self.getType(section, option) == 'directory': value = soft_chroot.chroot2abs(value) if self.getType(section, option) == 'directories': import json value = json.loads(value) if not (value and isinstance(value, list)): value = [] value = map(soft_chroot.chroot2abs, value) value = self.directories_delimiter.join(value) # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single=True) self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single=True) fireEvent('setting.save.%s.*.after' % section, single=True) return {'success': True} def isSectionReadable(self, section): meta = 'section_hidden' + self.optionMetaSuffix() try: return not self.p.getboolean(section, meta) except: pass # by default - every section is readable: return True def isOptionReadable(self, section, option): meta = option + self.optionMetaSuffix() if self.p.has_option(section, meta): meta_v = self.p.get(section, meta).lower() return (meta_v == 'rw') or (meta_v == 'ro') # by default - all is writable: return True def optionReadableCheckAndWarn(self, section, option): x = self.isOptionReadable(section, option) if not x: self.log.warning('Option "%s.%s" isn\'t readable', (section, option)) return x def isOptionWritable(self, section, option): meta = option + self.optionMetaSuffix() if self.p.has_option(section, meta): return self.p.get(section, meta).lower() == 'rw' # by default - all is writable: return True def optionMetaSuffix(self): return '_internal_meta' def isOptionMeta(self, section, option): """ A helper method for detecting internal-meta options in the ini-file For a meta options used following names: * section_hidden_internal_meta = (True | False) - for section visibility * <OPTION>_internal_meta = (ro|rw|hidden) - for section visibility """ suffix = self.optionMetaSuffix() return option.endswith(suffix) def getProperty(self, identifier): from couchpotato import get_db db = get_db() prop = None identifier = identifier.encode( "ascii", "ignore" ) # if identifier is not ascii it crashes below in the db access try: propert = db.get('property', identifier, with_doc=True) prop = propert['doc']['value'] except ValueError: propert = db.get('property', identifier) fireEvent('database.delete_corrupted', propert.get('_id')) except: self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0))) return prop def setProperty(self, identifier, value=''): from couchpotato import get_db db = get_db() try: p = db.get('property', identifier, with_doc=True) p['doc'].update({ 'identifier': identifier, 'value': toUnicode(value), }) db.update(p['doc']) except: db.insert({ '_t': 'property', 'identifier': identifier, 'value': toUnicode(value), })
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.helpers.encoding import toUnicode, ss from couchpotato.core.helpers.request import jsonified from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release from couchpotato.environment import Env import errno import os import re import shutil import traceback log = CPLog(__name__) class Renamer(Plugin): renaming_started = False checking_snatched = False def __init__(self): addApiView('renamer.scan', self.scanView, docs={ 'desc': 'For the renamer to check for new files to rename', })
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' # Do db stuff db_path = os.path.join(data_dir, 'couchpotato.db') # Backup before start and cleanup old databases new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time()))) # Create path and copy if not os.path.isdir(new_backup): os.makedirs(new_backup) src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal'] for src_file in src_files: if os.path.isfile(src_file): shutil.copy2(src_file, os.path.join(new_backup, os.path.basename(src_file))) # Remove older backups, keep backups 3 days or at least 3 backups = [] for directory in os.listdir(os.path.dirname(new_backup)): backup = os.path.join(os.path.dirname(new_backup), directory) if os.path.isdir(backup): backups.append(backup) total_backups = len(backups) for backup in backups: if total_backups > 3: if int(os.path.basename(backup)) < time.time() - 259200: for src_file in src_files: b_file = os.path.join(backup, os.path.basename(src_file)) if os.path.isfile(b_file): os.remove(b_file) os.rmdir(backup) total_backups -= 1 # Register environment settings Env.set('encoding', encoding) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + db_path) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Load configs & plugins loader = Env.get('loader') loader.preload(root = base_path) loader.run() # Load migrations initialize = True db = Env.get('db_path') if os.path.isfile(db_path): initialize = False from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version = latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not debug: log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() if initialize: fireEvent('app.initialize', in_order = True) # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else '' # Basic config app.secret_key = api_key # app.debug = development config = { 'use_reloader': reloader, 'host': Env.setting('host', default = '0.0.0.0'), 'port': tryInt(Env.setting('port', default = 5000)) } # Static path app.static_folder = os.path.join(base_path, 'couchpotato', 'static') web.add_url_rule('api/%s/static/<path:filename>' % api_key, endpoint = 'static', view_func = app.send_static_file) # Register modules app.register_blueprint(web, url_prefix = '%s/' % url_base) app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key)) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') # Go go go! web_container = WSGIContainer(app) web_container._log = _log loop = IOLoop.instance() application = Application([ (r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler), (r'.*', FallbackHandler, dict(fallback = web_container)), ], log_function = lambda x : None, debug = config['use_reloader'] ) try_restart = True restart_tries = 5 while try_restart: try: application.listen(config['port'], config['host'], no_keep_alive = True) loop.start() except Exception, e: try: nr, msg = e if nr == 48: log.info('Already in use, try %s more time after few seconds', restart_tries) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
from couchpotato.core.helpers.variable import splitString from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from urllib2 import URLError import base64 import json import socket import traceback import urllib log = CPLog(__name__) class XBMC(Notification): listen_to = ['renamer.after', 'movie.snatched'] use_json_notifications = {} http_time_between_calls = 0 def notify(self, message='', data=None, listener=None): if not data: data = {} hosts = splitString(self.conf('host')) successful = 0 max_successful = 0 for host in hosts: if self.use_json_notifications.get(host) is None: self.getXBMCJSONversion(host, message=message)
from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog import json import requests log = CPLog(__name__) class Synology(Downloader): protocol = ['nzb', 'torrent', 'torrent_magnet'] log = CPLog(__name__) def download(self, data = None, movie = None, filedata = None): if not movie: movie = {} if not data: data = {} response = False log.error('Sending "%s" (%s) to Synology.', (data['name'], data['protocol'])) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False try: # Send request to Synology srpc = SynologyRPC(host[0], host[1], self.conf('username'), self.conf('password')) if data['protocol'] == 'torrent_magnet':
from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 from multipartpost import MultipartPostHandler import cookielib import httplib import json import os import re import stat import time import urllib import urllib2 log = CPLog(__name__) class uTorrent(Downloader): protocol = ['torrent', 'torrent_magnet'] utorrent_api = None def connect(self): # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False self.utorrent_api = uTorrentAPI(host[0], port = host[1], username = self.conf('username'), password = self.conf('password'))
class Deluge(DownloaderBase): protocol = ['torrent', 'torrent_magnet'] log = CPLog(__name__) drpc = None def connect(self, reconnect=False): """ Connect to the delugeRPC, re-use connection when already available :param reconnect: force reconnect :return: DelugeRPC instance """ # Load host from config and split out port. host = cleanHost(self.conf('host'), protocol=False).split(':') # Force host assignment if len(host) == 1: host.append(80) if not isInt(host[1]): log.error( 'Config properties are not filled in correctly, port is missing.' ) return False if not self.drpc or reconnect: self.drpc = DelugeRPC(host[0], port=host[1], username=self.conf('username'), password=self.conf('password')) return self.drpc def download(self, data=None, media=None, filedata=None): """ Send a torrent/nzb file to the downloader :param data: dict returned from provider Contains the release information :param media: media dict with information Used for creating the filename when possible :param filedata: downloaded torrent/nzb filedata The file gets downloaded in the searcher and send to this function This is done to have failed checking before using the downloader, so the downloader doesn't need to worry about that :return: boolean One faile returns false, but the downloaded should log his own errors """ if not media: media = {} if not data: data = {} log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol'))) if not self.connect(): return False if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False # Set parameters for Deluge options = { 'add_paused': self.conf('paused', default=0), 'label': self.conf('label') } if self.conf('directory'): #if os.path.isdir(self.conf('directory')): options['download_location'] = self.conf('directory') #else: # log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) if self.conf('completed_directory'): #if os.path.isdir(self.conf('completed_directory')): options['move_completed'] = 1 options['move_completed_path'] = self.conf('completed_directory') #else: # log.error('Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) if data.get('seed_ratio'): options['stop_at_ratio'] = 1 options['stop_ratio'] = tryFloat(data.get('seed_ratio')) # Deluge only has seed time as a global option. Might be added in # in a future API release. # if data.get('seed_time'): # Send request to Deluge if data.get('protocol') == 'torrent_magnet': remote_torrent = self.drpc.add_torrent_magnet( data.get('url'), options) else: filename = self.createFileName(data, filedata, media) remote_torrent = self.drpc.add_torrent_file( filename, filedata, options) if not remote_torrent: log.error('Failed sending torrent to Deluge') return False log.info('Torrent sent to Deluge successfully.') return self.downloadReturnId(remote_torrent) def test(self): """ Check if connection works :return: bool """ if self.connect(True) and self.drpc.test(): return True return False def getAllDownloadStatus(self, ids): """ Get status of all active downloads :param ids: list of (mixed) downloader ids Used to match the releases for this downloader as there could be other downloaders active that it should ignore :return: list of releases """ log.debug('Checking Deluge download status.') if not self.connect(): return [] release_downloads = ReleaseDownloadList(self) queue = self.drpc.get_alltorrents(ids) if not queue: log.debug('Nothing in queue or error') return [] for torrent_id in queue: torrent = queue[torrent_id] if not 'hash' in torrent: # When given a list of ids, deluge will return an empty item for a non-existant torrent. continue log.debug( 'name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_on_completed'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) # Deluge has no easy way to work out if a torrent is stalled or failing. #status = 'failed' status = 'busy' # If an user opts to seed a torrent forever (usually associated to private trackers usage), stop_ratio will be 0 or -1 (depending on Deluge version). # In this scenario the status of the torrent would never change from BUSY to SEEDING. # The last check takes care of this case. if torrent['is_seed'] and ( (tryFloat(torrent['ratio']) < tryFloat(torrent['stop_ratio'])) or (tryFloat(torrent['stop_ratio']) < 0)): # We have torrent['seeding_time'] to work out what the seeding time is, but we do not # have access to the downloader seed_time, as with deluge we have no way to pass it # when the torrent is added. So Deluge will only look at the ratio. # See above comment in download(). status = 'seeding' elif torrent['is_seed'] and torrent['is_finished'] and torrent[ 'paused'] and torrent['state'] == 'Paused': status = 'completed' download_dir = sp(torrent['save_path']) if torrent['move_on_completed']: download_dir = torrent['move_completed_path'] torrent_files = [] for file_item in torrent['files']: torrent_files.append( sp(os.path.join(download_dir, file_item['path']))) release_downloads.append({ 'id': torrent['hash'], 'name': torrent['name'], 'status': status, 'original_status': torrent['state'], 'seed_ratio': torrent['ratio'], 'timeleft': str(timedelta(seconds=torrent['eta'])), 'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])), 'files': torrent_files, }) return release_downloads def pause(self, release_download, pause=True): if pause: return self.drpc.pause_torrent([release_download['id']]) else: return self.drpc.resume_torrent([release_download['id']]) def removeFailed(self, release_download): log.info('%s failed downloading, deleting...', release_download['name']) return self.drpc.remove_torrent(release_download['id'], True) def processComplete(self, release_download, delete_files=False): log.debug( 'Requesting Deluge to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.drpc.remove_torrent(release_download['id'], remove_local_data=delete_files)
def runCouchPotato(options, base_path, args, desktop = None): # Load settings from couchpotato.environment import Env settings = Env.get('settings') settings.setFile(options.config_file) # Create data dir if needed data_dir = os.path.expanduser(Env.setting('data_dir')) if data_dir == '': data_dir = getDataDir() if not os.path.isdir(data_dir): os.makedirs(data_dir) # Create logging dir log_dir = os.path.join(data_dir, 'logs'); if not os.path.isdir(log_dir): os.mkdir(log_dir) try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): pass # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' # Register environment settings Env.set('encoding', encoding) Env.set('uses_git', not options.nogit) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db')) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Disable server access log server_log = logging.getLogger('werkzeug') server_log.disabled = True # Only run once when debugging fire_load = False if os.environ.get('WERKZEUG_RUN_MAIN') or not debug or Env.get('desktop') or options.daemon: # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s' % options) # Load configs & plugins loader = Env.get('loader') loader.preload(root = base_path) loader.run() # Load migrations from migrate.versioning.api import version_control, db_version, version, upgrade db = Env.get('db_path') repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') logging.getLogger('migrate').setLevel(logging.WARNING) # Disable logging for migration latest_db_version = version(repo) initialize = True try: current_db_version = db_version(db, repo) initialize = False except: version_control(db, repo, version = latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not debug: log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() if initialize: fireEvent('app.initialize') fire_load = True # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else '' reloader = debug is True and not Env.get('desktop') and not options.daemon # Basic config app.secret_key = api_key config = { 'use_reloader': reloader, 'host': Env.setting('host', default = '0.0.0.0'), 'port': tryInt(Env.setting('port', default = 5000)) } # Static path web.add_url_rule('static/<path:filename>', endpoint = 'static', view_func = app.send_static_file) # Register modules app.register_blueprint(web, url_prefix = '%s/' % url_base) app.register_blueprint(api, url_prefix = '%s/%s/' % (url_base, api_key)) # Some logging and fire load event try: log.info('Starting server on port %(port)s' % config) except: pass if fire_load: fireEventAsync('app.load') # Go go go! try: app.run(**config) except (KeyboardInterrupt, SystemExit): raise except: log.error('Failed starting: %s' % traceback.format_exc()) raise
from apscheduler.scheduler import Scheduler as Sched from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin import logging log = CPLog(__name__) class Scheduler(Plugin): crons = {} intervals = {} started = False def __init__(self): addEvent('schedule.cron', self.cron) addEvent('schedule.interval', self.interval) addEvent('schedule.start', self.start) addEvent('schedule.restart', self.start) addEvent('app.load', self.start) self.sched = Sched(misfire_grace_time = 60) def remove(self, identifier): for type in ['interval', 'cron']: try: self.sched.unschedule_job(getattr(self, type)[identifier]['job']) log.debug('%s unscheduled %s', (type.capitalize(), identifier))
class Loader(object): do_restart = False def __init__(self): # Get options via arg from couchpotato.runner import getOptions self.options = getOptions(base_path, sys.argv[1:]) # Load settings settings = Env.get('settings') settings.setFile(self.options.config_file) # Create data dir if needed self.data_dir = os.path.expanduser(Env.setting('data_dir')) if self.data_dir == '': self.data_dir = getDataDir() if not os.path.isdir(self.data_dir): os.makedirs(self.data_dir) # Create logging dir self.log_dir = os.path.join(self.data_dir, 'logs'); if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir) # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') hdlr = handlers.RotatingFileHandler(os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr) def addSignals(self): signal.signal(signal.SIGINT, self.onExit) signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(1)) from couchpotato.core.event import addEvent addEvent('app.after_shutdown', self.afterShutdown) def afterShutdown(self, restart): self.do_restart = restart def onExit(self, signal, frame): from couchpotato.core.event import fireEvent fireEvent('app.crappy_shutdown', single = True) def run(self): self.addSignals() from couchpotato.runner import runCouchPotato runCouchPotato(self.options, base_path, sys.argv[1:], data_dir = self.data_dir, log_dir = self.log_dir, Env = Env) if self.do_restart: self.restart() def restart(self): try: # remove old pidfile first try: if self.runAsDaemon(): try: self.daemon.stop() except: pass except: self.log.critical(traceback.format_exc()) args = [sys.executable] + [os.path.join(base_path, __file__)] + sys.argv[1:] subprocess.Popen(args) except: self.log.critical(traceback.format_exc()) def daemonize(self): if self.runAsDaemon(): try: from daemon import Daemon self.daemon = Daemon(self.options.pid_file) self.daemon.daemonize() except SystemExit: raise except: self.log.critical(traceback.format_exc()) def runAsDaemon(self): return self.options.daemon and self.options.pid_file
class Deluge(Downloader): protocol = ['torrent', 'torrent_magnet'] log = CPLog(__name__) drpc = None def connect(self, reconnect=False): # Load host from config and split out port. host = cleanHost(self.conf('host'), protocol=False).split(':') if not isInt(host[1]): log.error( 'Config properties are not filled in correctly, port is missing.' ) return False if not self.drpc or reconnect: self.drpc = DelugeRPC(host[0], port=host[1], username=self.conf('username'), password=self.conf('password')) return self.drpc def download(self, data=None, media=None, filedata=None): if not media: media = {} if not data: data = {} log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol'))) if not self.connect(): return False if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False # Set parameters for Deluge options = { 'add_paused': self.conf('paused', default=0), 'label': self.conf('label') } if self.conf('directory'): if os.path.isdir(self.conf('directory')): options['download_location'] = self.conf('directory') else: log.error( 'Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) if self.conf('completed_directory'): if os.path.isdir(self.conf('completed_directory')): options['move_completed'] = 1 options['move_completed_path'] = self.conf( 'completed_directory') else: log.error( 'Download directory from Deluge settings: %s doesn\'t exist', self.conf('directory')) if data.get('seed_ratio'): options['stop_at_ratio'] = 1 options['stop_ratio'] = tryFloat(data.get('seed_ratio')) # Deluge only has seed time as a global option. Might be added in # in a future API release. # if data.get('seed_time'): # Send request to Deluge if data.get('protocol') == 'torrent_magnet': remote_torrent = self.drpc.add_torrent_magnet( data.get('url'), options) else: filename = self.createFileName(data, filedata, media) remote_torrent = self.drpc.add_torrent_file( filename, filedata, options) if not remote_torrent: log.error('Failed sending torrent to Deluge') return False log.info('Torrent sent to Deluge successfully.') return self.downloadReturnId(remote_torrent) def test(self): if self.connect(True) and self.drpc.test(): return True return False def getAllDownloadStatus(self, ids): log.debug('Checking Deluge download status.') if not self.connect(): return [] release_downloads = ReleaseDownloadList(self) queue = self.drpc.get_alltorrents(ids) if not queue: log.debug('Nothing in queue or error') return [] for torrent_id in queue: torrent = queue[torrent_id] if not 'hash' in torrent: # When given a list of ids, deluge will return an empty item for a non-existant torrent. continue log.debug( 'name=%s / id=%s / save_path=%s / move_on_completed=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (torrent['name'], torrent['hash'], torrent['save_path'], torrent['move_on_completed'], torrent['move_completed_path'], torrent['hash'], torrent['progress'], torrent['state'], torrent['eta'], torrent['ratio'], torrent['stop_ratio'], torrent['is_seed'], torrent['is_finished'], torrent['paused'])) # Deluge has no easy way to work out if a torrent is stalled or failing. #status = 'failed' status = 'busy' if torrent['is_seed'] and tryFloat(torrent['ratio']) < tryFloat( torrent['stop_ratio']): # We have torrent['seeding_time'] to work out what the seeding time is, but we do not # have access to the downloader seed_time, as with deluge we have no way to pass it # when the torrent is added. So Deluge will only look at the ratio. # See above comment in download(). status = 'seeding' elif torrent['is_seed'] and torrent['is_finished'] and torrent[ 'paused'] and torrent['state'] == 'Paused': status = 'completed' download_dir = sp(torrent['save_path']) if torrent['move_on_completed']: download_dir = torrent['move_completed_path'] torrent_files = [] for file_item in torrent['files']: torrent_files.append( sp(os.path.join(download_dir, file_item['path']))) release_downloads.append({ 'id': torrent['hash'], 'name': torrent['name'], 'status': status, 'original_status': torrent['state'], 'seed_ratio': torrent['ratio'], 'timeleft': str(timedelta(seconds=torrent['eta'])), 'folder': sp(download_dir if len(torrent_files) == 1 else os.path.join(download_dir, torrent['name'])), 'files': '|'.join(torrent_files), }) return release_downloads def pause(self, release_download, pause=True): if pause: return self.drpc.pause_torrent([release_download['id']]) else: return self.drpc.resume_torrent([release_download['id']]) def removeFailed(self, release_download): log.info('%s failed downloading, deleting...', release_download['name']) return self.drpc.remove_torrent(release_download['id'], True) def processComplete(self, release_download, delete_files=False): log.debug( 'Requesting Deluge to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.drpc.remove_torrent(release_download['id'], remove_local_data=delete_files)
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db')) # Backup before start and cleanup old databases new_backup = toUnicode(os.path.join(data_dir, 'db_backup', str(int(time.time())))) # Create path and copy if not os.path.isdir(new_backup): os.makedirs(new_backup) src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal'] for src_file in src_files: if os.path.isfile(src_file): dst_file = toUnicode(os.path.join(new_backup, os.path.basename(src_file))) shutil.copyfile(src_file, dst_file) # Try and copy stats seperately try: shutil.copystat(src_file, dst_file) except: pass # Remove older backups, keep backups 3 days or at least 3 backups = [] for directory in os.listdir(os.path.dirname(new_backup)): backup = toUnicode(os.path.join(os.path.dirname(new_backup), directory)) if os.path.isdir(backup): backups.append(backup) total_backups = len(backups) for backup in backups: if total_backups > 3: if tryInt(os.path.basename(backup)) < time.time() - 259200: for the_file in os.listdir(backup): file_path = os.path.join(backup, the_file) try: if os.path.isfile(file_path): os.remove(file_path) except: raise os.rmdir(backup) total_backups -= 1 # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db_path', toUnicode('sqlite:///' + db_path)) Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache'))) Env.set('cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python')))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Check if database exists db = Env.get('db_path') db_exists = os.path.isfile(toUnicode(db_path)) # Load migrations if db_exists: from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version = latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version: if development: log.error('There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.') else: log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default = '0.0.0.0') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } # Load the app application = Application([], log_function = lambda x : None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, login_url = '%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers(".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': toUnicode(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = toUnicode(base_path)) loader.run() # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order = True) # Go go go! from tornado.ioloop import IOLoop loop = IOLoop.current() # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { "certfile": config['ssl_cert'], "keyfile": config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) try_restart = True restart_tries = 5 while try_restart: try: server.listen(config['port'], config['host']) loop.start() except Exception, e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e if nr == 48: log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries)) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
def runCouchPotato(options, base_path, args, data_dir=None, log_dir=None, Env=None, desktop=None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = toUnicode(os.path.join(data_dir, 'couchpotato.db')) # Backup before start and cleanup old databases new_backup = toUnicode( os.path.join(data_dir, 'db_backup', str(int(time.time())))) if not os.path.isdir(new_backup): os.makedirs(new_backup) # Remove older backups, keep backups 3 days or at least 3 backups = [] for directory in os.listdir(os.path.dirname(new_backup)): backup = toUnicode(os.path.join(os.path.dirname(new_backup), directory)) if os.path.isdir(backup): backups.append(backup) latest_backup = tryInt(os.path.basename( sorted(backups)[-1])) if len(backups) > 0 else 0 if latest_backup < time.time() - 3600: # Create path and copy src_files = [ options.config_file, db_path, db_path + '-shm', db_path + '-wal' ] for src_file in src_files: if os.path.isfile(src_file): dst_file = toUnicode( os.path.join(new_backup, os.path.basename(src_file))) shutil.copyfile(src_file, dst_file) # Try and copy stats seperately try: shutil.copystat(src_file, dst_file) except: pass total_backups = len(backups) for backup in backups: if total_backups > 3: if tryInt(os.path.basename(backup)) < time.time() - 259200: for the_file in os.listdir(backup): file_path = os.path.join(backup, the_file) try: if os.path.isfile(file_path): os.remove(file_path) except: raise os.rmdir(backup) total_backups -= 1 # Register environment settings Env.set('app_dir', toUnicode(base_path)) Env.set('data_dir', toUnicode(data_dir)) Env.set('log_path', toUnicode(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db_path', toUnicode('sqlite:///' + db_path)) Env.set('cache_dir', toUnicode(os.path.join(data_dir, 'cache'))) Env.set( 'cache', FileSystemCache(toUnicode(os.path.join(Env.get('cache_dir'), 'python')))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default=False, type='bool') Env.set('debug', debug) # Development development = Env.setting('development', default=False, type='bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get( 'desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file=None, line=None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Check if database exists db = Env.get('db_path') db_exists = os.path.isfile(toUnicode(db_path)) # Load migrations if db_exists: from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version=latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version: if development: log.error( 'There is a database migration ready, but you are running development mode, so it won\'t be used. If you see this, you are stupid. Please disable development mode.' ) else: log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default='0.0.0.0') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default=5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default=None), 'ssl_key': Env.setting('ssl_key', default=None), } # Load the app application = Application( [], log_function=lambda x: None, debug=config['use_reloader'], gzip=True, cookie_secret=api_key, login_url='%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers( ".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, { "url": web_base + 'docs/' }), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers( ".*$", [('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, { 'path': toUnicode( os.path.join(base_path, 'couchpotato', 'static', dir_name)) })]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root=toUnicode(base_path)) loader.run() # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order=True) # Go go go! from tornado.ioloop import IOLoop loop = IOLoop.current() # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive=True, ssl_options={ "certfile": config['ssl_cert'], "keyfile": config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive=True) try_restart = True restart_tries = 5 while try_restart: try: server.listen(config['port'], config['host']) loop.start() except Exception, e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e if nr == 48: log.info( 'Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries)) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
class Settings(object): options = {} types = {} def __init__(self): addApiView( "settings", self.view, docs={ "desc": "Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.", "return": { "type": "object", "example": """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }""", }, }, ) addApiView( "settings.save", self.saveView, docs={ "desc": "Save setting to config file (settings.conf)", "params": { "section": {"desc": "The section name in settings.conf"}, "name": {"desc": "The option name"}, "value": {"desc": "The value you want to save"}, }, }, ) addEvent("database.setup", self.databaseSetup) self.file = None self.p = None self.log = None self.directories_delimiter = "::" def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def databaseSetup(self): fireEvent("database.setup_index", "property", PropertyIndex) def parser(self): return self.p def sections(self): res = filter(self.isSectionReadable, self.p.sections()) return res def connectEvents(self): addEvent("settings.options", self.addOptions) addEvent("settings.register", self.registerDefaults) addEvent("settings.save", self.save) def registerDefaults(self, section_name, options=None, save=True): if not options: options = {} self.addSection(section_name) for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get("default", "")) # Set UI-meta for option (hidden/ro/rw) if option.get("ui-meta"): value = option.get("ui-meta") if value: value = value.lower() if value in ["hidden", "rw", "ro"]: meta_option_name = option_name + self.optionMetaSuffix() self.setDefault(section_name, meta_option_name, value) else: self.log.warning( 'Wrong value for option %s.%s : ui-meta can not be equal to "%s"', (section_name, option_name, value), ) # Migrate old settings from old location to the new location if option.get("migrate_from"): if self.p.has_option(option.get("migrate_from"), option_name): previous_value = self.p.get(option.get("migrate_from"), option_name) self.p.set(section_name, option_name, previous_value) self.p.remove_option(option.get("migrate_from"), option_name) if option.get("type"): self.setType(section_name, option_name, option.get("type")) if save: self.save() def set(self, section, option, value): if not self.isOptionWritable(section, option): self.log.warning('set::option "%s.%s" isn\'t writable', (section, option)) return None if self.isOptionMeta(section, option): self.log.warning('set::option "%s.%s" cancelled, since it is a META option', (section, option)) return None return self.p.set(section, option, value) def get(self, option="", section="core", default=None, type=None): if self.isOptionMeta(section, option): self.log.warning('get::option "%s.%s" cancelled, since it is a META option', (section, option)) return None tp = type try: tp = self.getType(section, option) if not tp else tp if hasattr(self, "get%s" % tp.capitalize()): return getattr(self, "get%s" % tp.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def delete(self, option="", section="core"): if not self.isOptionWritable(section, option): self.log.warning('delete::option "%s.%s" isn\'t writable', (section, option)) return None if self.isOptionMeta(section, option): self.log.warning('set::option "%s.%s" cancelled, since it is a META option', (section, option)) return None self.p.remove_option(section, option) self.save() def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) == 1 def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryFloat(self.p.get(section, option)) def getDirectories(self, section, option): value = self.p.get(section, option) if value: return map(str.strip, str.split(value, self.directories_delimiter)) return [] def getUnicode(self, section, option): value = self.p.get(section, option).decode("unicode_escape") return toUnicode(value).strip() def getValues(self): from couchpotato.environment import Env values = {} soft_chroot = Env.get("softchroot") # TODO : There is two commented "continue" blocks (# COMMENTED_SKIPPING). They both are good... # ... but, they omit output of values of hidden and non-readable options # Currently, such behaviour could break the Web UI of CP... # So, currently this two blocks are commented (but they are required to # provide secure hidding of options. for section in self.sections(): # COMMENTED_SKIPPING # if not self.isSectionReadable(section): # continue values[section] = {} for option in self.p.items(section): (option_name, option_value) = option # skip meta options: if self.isOptionMeta(section, option_name): continue # COMMENTED_SKIPPING # if not self.isOptionReadable(section, option_name): # continue value = self.get(option_name, section) is_password = self.getType(section, option_name) == "password" if is_password and value: value = len(value) * "*" # chrootify directory before sending to UI: if (self.getType(section, option_name) == "directory") and value: try: value = soft_chroot.abs2chroot(value) except: value = "" # chrootify directories before sending to UI: if self.getType(section, option_name) == "directories": if not value: value = [] try: value = map(soft_chroot.abs2chroot, value) except: value = [] values[section][option_name] = value return values def save(self): with open(self.file, "wb") as configfile: self.p.write(configfile) def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def getType(self, section, option): tp = None try: tp = self.types[section][option] except: tp = "unicode" if not tp else tp return tp def addOptions(self, section_name, options): # no additional actions (related to ro-rw options) are required here if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): """Returns dict of UI-readable options To check, whether the option is readable self.isOptionReadable() is used """ res = {} # it is required to filter invisible options for UI, but also we should # preserve original tree for server's purposes. # So, next loops do one thing: copy options to res and in the process # 1. omit NON-READABLE (for UI) options, and # 2. put flags on READONLY options for section_key in self.options.keys(): section_orig = self.options[section_key] section_name = section_orig.get("name") if "name" in section_orig else section_key if self.isSectionReadable(section_name): section_copy = {} section_copy_groups = [] for section_field in section_orig: if section_field.lower() != "groups": section_copy[section_field] = section_orig[section_field] else: for group_orig in section_orig["groups"]: group_copy = {} group_copy_options = [] for group_field in group_orig: if group_field.lower() != "options": group_copy[group_field] = group_orig[group_field] else: for option in group_orig[group_field]: option_name = option.get("name") # You should keep in mind, that READONLY = !IS_WRITABLE # and IS_READABLE is a different thing if self.isOptionReadable(section_name, option_name): group_copy_options.append(option) if not self.isOptionWritable(section_name, option_name): option["readonly"] = True if len(group_copy_options) > 0: group_copy["options"] = group_copy_options section_copy_groups.append(group_copy) if len(section_copy_groups) > 0: section_copy["groups"] = section_copy_groups res[section_key] = section_copy return res def view(self, **kwargs): return {"options": self.getOptions(), "values": self.getValues()} def saveView(self, **kwargs): section = kwargs.get("section") option = kwargs.get("name") value = kwargs.get("value") if not self.isOptionWritable(section, option): self.log.warning('Option "%s.%s" isn\'t writable', (section, option)) return {"success": False} from couchpotato.environment import Env soft_chroot = Env.get("softchroot") if self.getType(section, option) == "directory": value = soft_chroot.chroot2abs(value) if self.getType(section, option) == "directories": import json value = json.loads(value) if not (value and isinstance(value, list)): value = [] value = map(soft_chroot.chroot2abs, value) value = self.directories_delimiter.join(value) # See if a value handler is attached, use that as value new_value = fireEvent("setting.save.%s.%s" % (section, option), value, single=True) self.set(section, option, (new_value if new_value else value).encode("unicode_escape")) self.save() # After save (for re-interval etc) fireEvent("setting.save.%s.%s.after" % (section, option), single=True) fireEvent("setting.save.%s.*.after" % section, single=True) return {"success": True} def isSectionReadable(self, section): meta = "section_hidden" + self.optionMetaSuffix() try: return not self.p.getboolean(section, meta) except: pass # by default - every section is readable: return True def isOptionReadable(self, section, option): meta = option + self.optionMetaSuffix() if self.p.has_option(section, meta): meta_v = self.p.get(section, meta).lower() return (meta_v == "rw") or (meta_v == "ro") # by default - all is writable: return True def optionReadableCheckAndWarn(self, section, option): x = self.isOptionReadable(section, option) if not x: self.log.warning('Option "%s.%s" isn\'t readable', (section, option)) return x def isOptionWritable(self, section, option): meta = option + self.optionMetaSuffix() if self.p.has_option(section, meta): return self.p.get(section, meta).lower() == "rw" # by default - all is writable: return True def optionMetaSuffix(self): return "_internal_meta" def isOptionMeta(self, section, option): """ A helper method for detecting internal-meta options in the ini-file For a meta options used following names: * section_hidden_internal_meta = (True | False) - for section visibility * <OPTION>_internal_meta = (ro|rw|hidden) - for section visibility """ suffix = self.optionMetaSuffix() return option.endswith(suffix) def getProperty(self, identifier): from couchpotato import get_db db = get_db() prop = None try: propert = db.get("property", identifier, with_doc=True) prop = propert["doc"]["value"] except ValueError: propert = db.get("property", identifier) fireEvent("database.delete_corrupted", propert.get("_id")) except: self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0))) return prop def setProperty(self, identifier, value=""): from couchpotato import get_db db = get_db() try: p = db.get("property", identifier, with_doc=True) p["doc"].update({"identifier": identifier, "value": toUnicode(value)}) db.update(p["doc"]) except: db.insert({"_t": "property", "identifier": identifier, "value": toUnicode(value)})
from bs4 import BeautifulSoup from couchpotato.core.event import fireEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ simplifyString from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt, getTitle from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse import re import time import traceback import xml.etree.ElementTree as XMLTree log = CPLog(__name__) class NzbIndex(NZBProvider, RSS): urls = { 'download': 'http://www.nzbindex.nl/download/', 'api': 'http://www.nzbindex.nl/rss/', } http_time_between_calls = 1 # Seconds def search(self, movie, quality): results = [] if self.isDisabled():
from apscheduler.scheduler import Scheduler as Sched from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin import logging log = CPLog(__name__) class Scheduler(Plugin): crons = {} intervals = {} started = False def __init__(self): logging.getLogger('apscheduler').setLevel(logging.ERROR) addEvent('schedule.cron', self.cron) addEvent('schedule.interval', self.interval) addEvent('schedule.start', self.start) addEvent('schedule.restart', self.start) addEvent('app.load', self.start) self.sched = Sched(misfire_grace_time = 60) def remove(self, identifier): for type in ['interval', 'cron']: try:
from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 from multipartpost import MultipartPostHandler import cookielib import httplib import json import os import re import stat import time import urllib import urllib2 log = CPLog(__name__) class uTorrent(Downloader): protocol = ['torrent', 'torrent_magnet'] utorrent_api = None def connect(self): # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error( 'Config properties are not filled in correctly, port is missing.' ) return False
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): pass # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' # Register environment settings Env.set('encoding', encoding) Env.set('uses_git', not options.nogit) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + os.path.join(data_dir, 'couchpotato.db')) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) if not development: atexit.register(cleanup) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Disable server access log logging.getLogger('werkzeug').setLevel(logging.WARNING) # Only run once when debugging fire_load = False if os.environ.get('WERKZEUG_RUN_MAIN') or not reloader: # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s' % options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s' % (category, message, filename, lineno)) warnings.showwarning = customwarn # Load configs & plugins loader = Env.get('loader') loader.preload(root = base_path) loader.run() # Load migrations initialize = True db = Env.get('db_path') if os.path.isfile(db.replace('sqlite:///', '')): initialize = False from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') logging.getLogger('migrate').setLevel(logging.WARNING) # Disable logging for migration latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version = latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not debug: log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() if initialize: fireEvent('app.initialize', in_order = True) fire_load = True # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else '' # Basic config app.secret_key = api_key config = { 'use_reloader': reloader, 'host': Env.setting('host', default = '0.0.0.0'), 'port': tryInt(Env.setting('port', default = 5000)) } # Static path app.static_folder = os.path.join(base_path, 'couchpotato', 'static') web.add_url_rule('%s/static/<path:filename>' % api_key, endpoint = 'static', view_func = app.send_static_file) # Register modules app.register_blueprint(web, url_prefix = '%s/' % url_base) app.register_blueprint(api, url_prefix = '%s/%s/' % (url_base, api_key)) # Some logging and fire load event try: log.info('Starting server on port %(port)s' % config) except: pass if fire_load: fireEventAsync('app.load') # Go go go! try_restart = True restart_tries = 5 while try_restart: try: app.run(**config) except Exception, e: try: nr, msg = e if nr == 48: log.info('Already in use, try %s more time after few seconds' % restart_tries) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
class Loader(object): do_restart = False def __init__(self): # Get options via arg from couchpotato.runner import getOptions self.options = getOptions(sys.argv[1:]) # Load settings settings = Env.get('settings') settings.setFile(self.options.config_file) # Create data dir if needed if self.options.data_dir: self.data_dir = self.options.data_dir else: self.data_dir = os.path.expanduser(Env.setting('data_dir')) if self.data_dir == '': self.data_dir = getDataDir() if not os.path.isdir(self.data_dir): os.makedirs(self.data_dir) # Create logging dir self.log_dir = os.path.join(self.data_dir, 'logs') if not os.path.isdir(self.log_dir): os.makedirs(self.log_dir) # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') hdlr = handlers.RotatingFileHandler( os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr) def addSignals(self): signal.signal(signal.SIGINT, self.onExit) signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(1)) from couchpotato.core.event import addEvent addEvent('app.after_shutdown', self.afterShutdown) def afterShutdown(self, restart): self.do_restart = restart def onExit(self, signal, frame): from couchpotato.core.event import fireEvent fireEvent('app.shutdown', single=True) def run(self): self.addSignals() from couchpotato.runner import runCouchPotato runCouchPotato(self.options, base_path, sys.argv[1:], data_dir=self.data_dir, log_dir=self.log_dir, Env=Env) if self.do_restart: self.restart() def restart(self): try: # remove old pidfile first try: if self.runAsDaemon(): try: self.daemon.stop() except: pass except: self.log.critical(traceback.format_exc()) # Release log files and shutdown logger logging.shutdown() time.sleep(3) args = [sys.executable] + [ os.path.join(base_path, os.path.basename(__file__)) ] + sys.argv[1:] subprocess.Popen(args) except: self.log.critical(traceback.format_exc()) def daemonize(self): if self.runAsDaemon(): try: from daemon import Daemon self.daemon = Daemon(self.options.pid_file) self.daemon.daemonize() except SystemExit: raise except: self.log.critical(traceback.format_exc()) def runAsDaemon(self): return self.options.daemon and self.options.pid_file
from axl.axel import Event from couchpotato.core.helpers.variable import mergeDicts, natcmp from couchpotato.core.logger import CPLog import threading import traceback log = CPLog(__name__) events = {} def runHandler(name, handler, *args, **kwargs): try: return handler(*args, **kwargs) except: from couchpotato.environment import Env log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all())) def addEvent(name, handler, priority = 100): if events.get(name): e = events[name] else: e = events[name] = Event(name = name, threads = 20, exc_info = True, traceback = True, lock = threading.RLock()) def createHandle(*args, **kwargs): try: parent = handler.im_self bc = hasattr(parent, 'beforeCall') if bc: parent.beforeCall(handler) h = runHandler(name, handler, *args, **kwargs) ac = hasattr(parent, 'afterCall')
class Settings(object): options = {} types = {} def __init__(self): addApiView('settings', self.view, docs = { 'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.', 'return': {'type': 'object', 'example': """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }"""} }) addApiView('settings.save', self.saveView, docs = { 'desc': 'Save setting to config file (settings.conf)', 'params': { 'section': {'desc': 'The section name in settings.conf'}, 'name': {'desc': 'The option name'}, 'value': {'desc': 'The value you want to save'}, } }) addEvent('database.setup', self.databaseSetup) self.file = None self.p = None self.log = None def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def databaseSetup(self): fireEvent('database.setup_index', 'property', PropertyIndex) def parser(self): return self.p def sections(self): return self.p.sections() def connectEvents(self): addEvent('settings.options', self.addOptions) addEvent('settings.register', self.registerDefaults) addEvent('settings.save', self.save) def registerDefaults(self, section_name, options = None, save = True): if not options: options = {} self.addSection(section_name) for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get('default', '')) # Migrate old settings from old location to the new location if option.get('migrate_from'): if self.p.has_option(option.get('migrate_from'), option_name): previous_value = self.p.get(option.get('migrate_from'), option_name) self.p.set(section_name, option_name, previous_value) self.p.remove_option(option.get('migrate_from'), option_name) if option.get('type'): self.setType(section_name, option_name, option.get('type')) if save: self.save() def set(self, section, option, value): return self.p.set(section, option, value) def get(self, option = '', section = 'core', default = None, type = None): try: try: type = self.types[section][option] except: type = 'unicode' if not type else type if hasattr(self, 'get%s' % type.capitalize()): return getattr(self, 'get%s' % type.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def delete(self, option = '', section = 'core'): self.p.remove_option(section, option) self.save() def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) == 1 def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryFloat(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option).decode('unicode_escape') return toUnicode(value).strip() def getValues(self): values = {} for section in self.sections(): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option is_password = False try: is_password = self.types[section][option_name] == 'password' except: pass values[section][option_name] = self.get(option_name, section) if is_password and values[section][option_name]: values[section][option_name] = len(values[section][option_name]) * '*' return values def save(self): with open(self.file, 'wb') as configfile: self.p.write(configfile) self.log.debug('Saved settings') def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def addOptions(self, section_name, options): if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): return self.options def view(self, **kwargs): return { 'options': self.getOptions(), 'values': self.getValues() } def saveView(self, **kwargs): section = kwargs.get('section') option = kwargs.get('name') value = kwargs.get('value') # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True) self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single = True) fireEvent('setting.save.%s.*.after' % section, single = True) return { 'success': True, } def getProperty(self, identifier): from couchpotato import get_db db = get_db() prop = None try: propert = db.get('property', identifier, with_doc = True) prop = propert['doc']['value'] except: pass # self.log.debug('Property "%s" doesn\'t exist: %s', (identifier, traceback.format_exc(0))) return prop def setProperty(self, identifier, value = ''): from couchpotato import get_db db = get_db() try: p = db.get('property', identifier, with_doc = True) p['doc'].update({ 'identifier': identifier, 'value': toUnicode(value), }) db.update(p['doc']) except: db.insert({ '_t': 'property', 'identifier': identifier, 'value': toUnicode(value), })
from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.logger import CPLog from hashlib import sha1 from multipartpost import MultipartPostHandler from datetime import timedelta import os import cookielib import httplib import json import re import time import urllib import urllib2 log = CPLog(__name__) class uTorrent(Downloader): type = ['torrent', 'torrent_magnet'] utorrent_api = None def download(self, data, movie, filedata = None): log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.')
from base64 import standard_b64encode from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import ss from couchpotato.core.helpers.variable import tryInt, md5 from couchpotato.core.logger import CPLog from datetime import timedelta import re import shutil import socket import traceback import xmlrpclib log = CPLog(__name__) class NZBGet(Downloader): type = ['nzb'] url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc' def download(self, data={}, movie={}, filedata=None): if not filedata: log.error('Unable to get NZB file: %s', traceback.format_exc()) return False log.info('Sending "%s" to NZBGet.', data.get('name')) url = self.url % { 'host': self.conf('host'),
class Settings(object): options = {} types = {} def __init__(self): addApiView('settings', self.view, docs = { 'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.', 'return': {'type': 'object', 'example': """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }"""} }) addApiView('settings.save', self.saveView, docs = { 'desc': 'Save setting to config file (settings.conf)', 'params': { 'section': {'desc': 'The section name in settings.conf'}, 'option': {'desc': 'The option name'}, 'value': {'desc': 'The value you want to save'}, } }) def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def parser(self): return self.p def sections(self): return self.p.sections() def connectEvents(self): addEvent('settings.options', self.addOptions) addEvent('settings.register', self.registerDefaults) addEvent('settings.save', self.save) def registerDefaults(self, section_name, options = None, save = True): if not options: options = {} self.addSection(section_name) for option_name, option in options.items(): self.setDefault(section_name, option_name, option.get('default', '')) # Migrate old settings from old location to the new location if option.get('migrate_from'): if self.p.has_option(option.get('migrate_from'), option_name): previous_value = self.p.get(option.get('migrate_from'), option_name) self.p.set(section_name, option_name, previous_value) self.p.remove_option(option.get('migrate_from'), option_name) if option.get('type'): self.setType(section_name, option_name, option.get('type')) if save: self.save() def set(self, section, option, value): return self.p.set(section, option, value) def get(self, option = '', section = 'core', default = None, type = None): try: try: type = self.types[section][option] except: type = 'unicode' if not type else type if hasattr(self, 'get%s' % type.capitalize()): return getattr(self, 'get%s' % type.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def delete(self, option = '', section = 'core'): self.p.remove_option(section, option) self.save() def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) == 1 def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryFloat(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option).decode('unicode_escape') return toUnicode(value).strip() def getValues(self): values = {} for section in self.sections(): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option values[section][option_name] = self.get(option_name, section) return values def save(self): with open(self.file, 'wb') as configfile: self.p.write(configfile) self.log.debug('Saved settings') def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def addOptions(self, section_name, options): if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): return self.options def view(self, **kwargs): return { 'options': self.getOptions(), 'values': self.getValues() } def saveView(self, **kwargs): section = kwargs.get('section') option = kwargs.get('name') value = kwargs.get('value') # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True) self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single = True) fireEvent('setting.save.%s.*.after' % section, single = True) return { 'success': True, } def getProperty(self, identifier): from couchpotato import get_session db = get_session() prop = None try: propert = db.query(Properties).filter_by(identifier = identifier).first() prop = propert.value except: pass return prop def setProperty(self, identifier, value = ''): from couchpotato import get_session try: db = get_session() p = db.query(Properties).filter_by(identifier = identifier).first() if not p: p = Properties() db.add(p) p.identifier = identifier p.value = toUnicode(value) db.commit() except: self.log.error('Failed: %s', traceback.format_exc()) db.rollback() finally: db.close()
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = sp(os.path.join(data_dir, 'database')) # Check if database exists db = SuperThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups backup_path = sp(os.path.join(data_dir, 'db_backup')) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): for backup_file in sorted(files): ints = re.findall('\d+', backup_file) # Delete non zip files if len(ints) != 1: os.remove(os.path.join(backup_path, backup_file)) else: existing_backups.append((int(ints[0]), backup_file)) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: os.remove(os.path.join(backup_path, eb[1])) # Create new backup new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') for root, dirs, files in os.walk(db_path): for zfilename in files: zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename)) zipf.close() # Open last db.open() else: db.create() # Force creation of cachedir log_dir = sp(log_dir) cache_dir = sp(os.path.join(data_dir, 'cache')) python_cache = sp(os.path.join(cache_dir, 'python')) if not os.path.exists(cache_dir): os.mkdir(cache_dir) if not os.path.exists(python_cache): os.mkdir(python_cache) # Register environment settings Env.set('app_dir', sp(base_path)) Env.set('data_dir', sp(data_dir)) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) Env.set('http_opener', requests.Session()) Env.set('cache_dir', cache_dir) Env.set('cache', FileSystemCache(python_cache)) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding')) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors # noinspection PyUnresolvedReferences import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') if not api_key: api_key = uuid4().hex Env.setting('api_key', value = api_key) api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default = '0.0.0.0') # app.debug = development config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } # Load the app application = Application( [], log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, login_url = '%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers(".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = sp(base_path)) loader.run() # Fill database with needed stuff fireEvent('database.setup') if not db_exists: fireEvent('app.initialize', in_order = True) fireEvent('app.migrate') # Go go go! from tornado.ioloop import IOLoop from tornado.autoreload import add_reload_hook loop = IOLoop.current() # Reload hook def test(): fireEvent('app.shutdown') add_reload_hook(test) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') if config['ssl_cert'] and config['ssl_key']: server = HTTPServer(application, no_keep_alive = True, ssl_options = { 'certfile': config['ssl_cert'], 'keyfile': config['ssl_key'], }) else: server = HTTPServer(application, no_keep_alive = True) try_restart = True restart_tries = 5 while try_restart: try: server.listen(config['port'], config['host']) loop.start() except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) try: nr, msg = e if nr == 48: log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries)) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' # Do db stuff db_path = os.path.join(data_dir, 'couchpotato.db') # Backup before start and cleanup old databases new_backup = os.path.join(data_dir, 'db_backup', str(int(time.time()))) # Create path and copy if not os.path.isdir(new_backup): os.makedirs(new_backup) src_files = [options.config_file, db_path, db_path + '-shm', db_path + '-wal'] for src_file in src_files: if os.path.isfile(src_file): shutil.copy2(src_file, os.path.join(new_backup, os.path.basename(src_file))) # Remove older backups, keep backups 3 days or at least 3 backups = [] for directory in os.listdir(os.path.dirname(new_backup)): backup = os.path.join(os.path.dirname(new_backup), directory) if os.path.isdir(backup): backups.append(backup) total_backups = len(backups) for backup in backups: if total_backups > 3: if int(os.path.basename(backup)) < time.time() - 259200: for src_file in src_files: b_file = os.path.join(backup, os.path.basename(src_file)) if os.path.isfile(b_file): os.remove(b_file) os.rmdir(backup) total_backups -= 1 # Register environment settings Env.set('encoding', encoding) Env.set('app_dir', base_path) Env.set('data_dir', data_dir) Env.set('log_path', os.path.join(log_dir, 'CouchPotato.log')) Env.set('db_path', 'sqlite:///' + db_path) Env.set('cache_dir', os.path.join(data_dir, 'cache')) Env.set('cache', FileSystemCache(os.path.join(Env.get('cache_dir'), 'python'))) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp', 'migrate']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Check if database exists db = Env.get('db_path') db_exists = os.path.isfile(db_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = base_path) loader.run() # Load migrations if db_exists: from migrate.versioning.api import version_control, db_version, version, upgrade repo = os.path.join(base_path, 'couchpotato', 'core', 'migration') latest_db_version = version(repo) try: current_db_version = db_version(db, repo) except: version_control(db, repo, version = latest_db_version) current_db_version = db_version(db, repo) if current_db_version < latest_db_version and not development: log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version)) upgrade(db, repo) # Configure Database from couchpotato.core.settings.model import setup setup() # Fill database with needed stuff if not db_exists: fireEvent('app.initialize', in_order = True) # Create app from couchpotato import app api_key = Env.setting('api_key') url_base = '/' + Env.setting('url_base').lstrip('/') if Env.setting('url_base') else '' # Basic config app.secret_key = api_key # app.debug = development config = { 'use_reloader': reloader, 'host': Env.setting('host', default = '0.0.0.0'), 'port': tryInt(Env.setting('port', default = 5000)) } # Static path app.static_folder = os.path.join(base_path, 'couchpotato', 'static') web.add_url_rule('api/%s/static/<path:filename>' % api_key, endpoint = 'static', view_func = app.send_static_file) # Register modules app.register_blueprint(web, url_prefix = '%s/' % url_base) app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key)) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') # Go go go! from tornado.ioloop import IOLoop web_container = WSGIContainer(app) web_container._log = _log loop = IOLoop.instance() application = Application([ (r'%s/api/%s/nonblock/(.*)/' % (url_base, api_key), NonBlockHandler), (r'.*', FallbackHandler, dict(fallback = web_container)), ], log_function = lambda x : None, debug = config['use_reloader'] ) try_restart = True restart_tries = 5 while try_restart: try: application.listen(config['port'], config['host'], no_keep_alive = True) loop.start() except Exception, e: try: nr, msg = e if nr == 48: log.info('Already in use, try %s more time after few seconds', restart_tries) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except: pass raise try_restart = False
class Transmission(Downloader): protocol = ['torrent', 'torrent_magnet'] log = CPLog(__name__) trpc = None def connect(self, reconnect = False): # Load host from config and split out port. host = cleanHost(self.conf('host'), protocol = False).split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False if not self.trpc or reconnect: self.trpc = TransmissionRPC(host[0], port = host[1], rpc_url = self.conf('rpc_url').strip('/ '), username = self.conf('username'), password = self.conf('password')) return self.trpc def download(self, data = None, media = None, filedata = None): if not media: media = {} if not data: data = {} log.info('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('protocol'))) if not self.connect(): return False if not filedata and data.get('protocol') == 'torrent': log.error('Failed sending torrent, no data') return False # Set parameters for adding torrent params = { 'paused': self.conf('paused', default = False) } if self.conf('directory'): if os.path.isdir(self.conf('directory')): params['download-dir'] = self.conf('directory') else: log.error('Download directory from Transmission settings: %s doesn\'t exist', self.conf('directory')) # Change parameters of torrent torrent_params = {} if data.get('seed_ratio'): torrent_params['seedRatioLimit'] = tryFloat(data.get('seed_ratio')) torrent_params['seedRatioMode'] = 1 if data.get('seed_time'): torrent_params['seedIdleLimit'] = tryInt(data.get('seed_time')) * 60 torrent_params['seedIdleMode'] = 1 # Send request to Transmission if data.get('protocol') == 'torrent_magnet': remote_torrent = self.trpc.add_torrent_uri(data.get('url'), arguments = params) torrent_params['trackerAdd'] = self.torrent_trackers else: remote_torrent = self.trpc.add_torrent_file(b64encode(filedata), arguments = params) if not remote_torrent: log.error('Failed sending torrent to Transmission') return False # Change settings of added torrents if torrent_params: self.trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) log.info('Torrent sent to Transmission successfully.') return self.downloadReturnId(remote_torrent['torrent-added']['hashString']) def test(self): if self.connect(True) and self.trpc.get_session(): return True return False def getAllDownloadStatus(self, ids): log.debug('Checking Transmission download status.') if not self.connect(): return [] release_downloads = ReleaseDownloadList(self) return_params = { 'fields': ['id', 'name', 'hashString', 'percentDone', 'status', 'eta', 'isStalled', 'isFinished', 'downloadDir', 'uploadRatio', 'secondsSeeding', 'seedIdleLimit', 'files'] } session = self.trpc.get_session() queue = self.trpc.get_alltorrents(return_params) if not (queue and queue.get('torrents')): log.debug('Nothing in queue or error') return [] for torrent in queue['torrents']: if torrent['hashString'] in ids: log.debug('name=%s / id=%s / downloadDir=%s / hashString=%s / percentDone=%s / status=%s / isStalled=%s / eta=%s / uploadRatio=%s / isFinished=%s / incomplete-dir-enabled=%s / incomplete-dir=%s', (torrent['name'], torrent['id'], torrent['downloadDir'], torrent['hashString'], torrent['percentDone'], torrent['status'], torrent.get('isStalled', 'N/A'), torrent['eta'], torrent['uploadRatio'], torrent['isFinished'], session['incomplete-dir-enabled'], session['incomplete-dir'])) status = 'busy' if torrent.get('isStalled') and not torrent['percentDone'] == 1 and self.conf('stalled_as_failed'): status = 'failed' elif torrent['status'] == 0 and torrent['percentDone'] == 1: status = 'completed' elif torrent['status'] in [5, 6]: status = 'seeding' if session['incomplete-dir-enabled'] and status == 'busy': torrent_folder = session['incomplete-dir'] else: torrent_folder = torrent['downloadDir'] torrent_files = [] for file_item in torrent['files']: torrent_files.append(sp(os.path.join(torrent_folder, file_item['name']))) release_downloads.append({ 'id': torrent['hashString'], 'name': torrent['name'], 'status': status, 'original_status': torrent['status'], 'seed_ratio': torrent['uploadRatio'], 'timeleft': str(timedelta(seconds = torrent['eta'])), 'folder': sp(torrent_folder if len(torrent_files) == 1 else os.path.join(torrent_folder, torrent['name'])), 'files': '|'.join(torrent_files) }) return release_downloads def pause(self, release_download, pause = True): if pause: return self.trpc.stop_torrent(release_download['id']) else: return self.trpc.start_torrent(release_download['id']) def removeFailed(self, release_download): log.info('%s failed downloading, deleting...', release_download['name']) return self.trpc.remove_torrent(release_download['id'], True) def processComplete(self, release_download, delete_files = False): log.debug('Requesting Transmission to remove the torrent %s%s.', (release_download['name'], ' and cleanup the downloaded files' if delete_files else '')) return self.trpc.remove_torrent(release_download['id'], delete_files)
class Settings(object): options = {} types = {} def __init__(self): addApiView( "settings", self.view, docs={ "desc": "Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.", "return": { "type": "object", "example": """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }""", }, }, ) addApiView( "settings.save", self.saveView, docs={ "desc": "Save setting to config file (settings.conf)", "params": { "section": {"desc": "The section name in settings.conf"}, "option": {"desc": "The option name"}, "value": {"desc": "The value you want to save"}, }, }, ) def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def parser(self): return self.p def sections(self): return self.p.sections() def connectEvents(self): addEvent("settings.options", self.addOptions) addEvent("settings.register", self.registerDefaults) addEvent("settings.save", self.save) def registerDefaults(self, section_name, options={}, save=True): self.addSection(section_name) for option_name, option in options.iteritems(): self.setDefault(section_name, option_name, option.get("default", "")) if option.get("type"): self.setType(section_name, option_name, option.get("type")) if save: self.save(self) def set(self, section, option, value): return self.p.set(section, option, value) def get(self, option="", section="core", default="", type=None): try: try: type = self.types[section][option] except: type = "unicode" if not type else type if hasattr(self, "get%s" % type.capitalize()): return getattr(self, "get%s" % type.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryInt(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option).decode("unicode_escape") return toUnicode(value).strip() def getValues(self): values = {} for section in self.sections(): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option values[section][option_name] = self.get(option_name, section) return values def save(self): with open(self.file, "wb") as configfile: self.p.write(configfile) self.log.debug("Saved settings") def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def addOptions(self, section_name, options): if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): return self.options def view(self): return jsonified({"options": self.getOptions(), "values": self.getValues()}) def saveView(self): params = getParams() section = params.get("section") option = params.get("name") value = params.get("value") # See if a value handler is attached, use that as value new_value = fireEvent("setting.save.%s.%s" % (section, option), value, single=True) self.set(section, option, (new_value if new_value else value).encode("unicode_escape")) self.save() return jsonified({"success": True}) def getProperty(self, identifier): from couchpotato import get_session db = get_session() try: prop = db.query(Properties).filter_by(identifier=identifier).first() return prop.value if prop else None except: return None def setProperty(self, identifier, value=""): from couchpotato import get_session db = get_session() p = db.query(Properties).filter_by(identifier=identifier).first() if not p: p = Properties() db.add(p) p.identifier = identifier p.value = toUnicode(value) db.commit()
class Settings(): options = {} types = {} def __init__(self): addApiView('settings', self.view) addApiView('settings.save', self.saveView) def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def parser(self): return self.p def sections(self): return self.p.sections() def connectEvents(self): addEvent('settings.options', self.addOptions) addEvent('settings.register', self.registerDefaults) addEvent('settings.save', self.save) def registerDefaults(self, section_name, options = {}, save = True): self.addSection(section_name) for option_name, option in options.iteritems(): self.setDefault(section_name, option_name, option.get('default', '')) if option.get('type'): self.setType(section_name, option_name, option.get('type')) if save: self.save(self) def set(self, section, option, value): return self.p.set(section, option, value) def get(self, option = '', section = 'core', default = '', type = None): try: try: type = self.types[section][option] except: type = 'unicode' if not type else type if hasattr(self, 'get%s' % type.capitalize()): return getattr(self, 'get%s' % type.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryInt(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option) return toUnicode(value).strip() def getValues(self): values = {} for section in self.sections(): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option values[section][option_name] = self.get(option_name, section) return values def save(self): with open(self.file, 'wb') as configfile: self.p.write(configfile) self.log.debug('Saved settings') def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def addOptions(self, section_name, options): if not self.options.get(section_name): self.options[section_name] = options else: options['groups'] = self.options[section_name].get('groups') + options.get('groups') self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): return self.options def view(self): return jsonified({ 'options': self.getOptions(), 'values': self.getValues() }) def saveView(self): params = getParams() section = params.get('section') option = params.get('name') value = params.get('value') # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True) self.set(section, option, new_value if new_value else value) self.save() return jsonified({ 'success': True, })
from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import tryUrlencode, ss from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from urllib2 import URLError from uuid import uuid4 import hashlib import httplib import json import socket import ssl import sys import traceback import urllib2 log = CPLog(__name__) class NZBVortex(Downloader): type = ['nzb'] api_level = None session_id = None def download(self, data = {}, movie = {}, manual = False, filedata = None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')) or not self.getApiLevel(): return # Send the nzb try: nzb_filename = self.createFileName(data, filedata, movie)
from couchpotato.core.helpers.request import getParams, jsonified from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ getImdb, link, symlink from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, File, Profile, Release, \ ReleaseInfo from couchpotato.environment import Env import errno import os import re import shutil import time import traceback log = CPLog(__name__) class Renamer(Plugin): renaming_started = False checking_snatched = False def __init__(self): addApiView('renamer.scan', self.scanView, docs = { 'desc': 'For the renamer to check for new files to rename in a folder', 'params': { 'movie_folder': {'desc': 'Optional: The folder of the movie to scan. Keep empty for default renamer folder.'}, 'downloader' : {'desc': 'Optional: The downloader this movie has been downloaded with'}, 'download_id': {'desc': 'Optional: The downloader\'s nzb/torrent ID'}, },
from base64 import b64encode from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog import httplib import json import os.path import re import urllib2 log = CPLog(__name__) class Transmission(Downloader): type = ['torrent', 'torrent_magnet'] log = CPLog(__name__) def download(self, data, movie, filedata = None): log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False # Set parameters for Transmission folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] folder_path = os.path.join(self.conf('directory', default = ''), folder_name).rstrip(os.path.sep)
class Transmission(Downloader): type = ['torrent', 'torrent_magnet'] log = CPLog(__name__) def download(self, data, movie, manual=False, filedata=None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error( 'Config properties are not filled in correctly, port is missing.' ) return False # Set parameters for Transmission folder_name = self.createFileName(data, filedata, movie)[:-len(data.get('type')) - 1] folder_path = os.path.join(self.conf('directory', default=''), folder_name).rstrip(os.path.sep) # Create the empty folder to download too self.makeDir(folder_path) params = { 'paused': self.conf('paused', default=0), 'download-dir': folder_path } torrent_params = { 'seedRatioLimit': self.conf('ratio'), 'seedRatioMode': (0 if self.conf('ratio') else 1) } if not filedata and data.get('type') == 'torrent': log.error('Failed sending torrent, no data') return False # Send request to Transmission try: trpc = TransmissionRPC(host[0], port=host[1], username=self.conf('username'), password=self.conf('password')) if data.get('type') == 'torrent_magnet': remote_torrent = trpc.add_torrent_uri(data.get('url'), arguments=params) torrent_params['trackerAdd'] = self.torrent_trackers else: remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments=params) # Change settings of added torrents trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params) return True except Exception, err: log.error('Failed to change settings for transfer: %s', err) return False
from bencode import bencode, bdecode from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import isInt, ss from couchpotato.core.logger import CPLog from datetime import timedelta from hashlib import sha1 from multipartpost import MultipartPostHandler import cookielib import httplib import json import re import time import urllib import urllib2 log = CPLog(__name__) class uTorrent(Downloader): type = ['torrent', 'torrent_magnet'] utorrent_api = None def download(self, data, movie, filedata=None): log.debug('Sending "%s" (%s) to uTorrent.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]):
class Settings(object): options = {} types = {} def __init__(self): addApiView('settings', self.view, docs = { 'desc': 'Return the options and its values of settings.conf. Including the default values and group ordering used on the settings page.', 'return': {'type': 'object', 'example': """{ // objects like in __init__.py of plugin "options": { "moovee" : { "groups" : [{ "description" : "SD movies only", "name" : "#alt.binaries.moovee", "options" : [{ "default" : false, "name" : "enabled", "type" : "enabler" }], "tab" : "providers" }], "name" : "moovee" } }, // object structured like settings.conf "values": { "moovee": { "enabled": false } } }"""} }) addApiView('settings.save', self.saveView, docs = { 'desc': 'Save setting to config file (settings.conf)', 'params': { 'section': {'desc': 'The section name in settings.conf'}, 'option': {'desc': 'The option name'}, 'value': {'desc': 'The value you want to save'}, } }) def setFile(self, config_file): self.file = config_file self.p = ConfigParser.RawConfigParser() self.p.read(config_file) from couchpotato.core.logger import CPLog self.log = CPLog(__name__) self.connectEvents() def parser(self): return self.p def sections(self): return self.p.sections() def connectEvents(self): addEvent('settings.options', self.addOptions) addEvent('settings.register', self.registerDefaults) addEvent('settings.save', self.save) def registerDefaults(self, section_name, options = {}, save = True): self.addSection(section_name) for option_name, option in options.iteritems(): self.setDefault(section_name, option_name, option.get('default', '')) if option.get('type'): self.setType(section_name, option_name, option.get('type')) if save: self.save(self) def set(self, section, option, value): return self.p.set(section, option, value) def get(self, option = '', section = 'core', default = None, type = None): try: try: type = self.types[section][option] except: type = 'unicode' if not type else type if hasattr(self, 'get%s' % type.capitalize()): return getattr(self, 'get%s' % type.capitalize())(section, option) else: return self.getUnicode(section, option) except: return default def getEnabler(self, section, option): return self.getBool(section, option) def getBool(self, section, option): try: return self.p.getboolean(section, option) except: return self.p.get(section, option) == 1 def getInt(self, section, option): try: return self.p.getint(section, option) except: return tryInt(self.p.get(section, option)) def getFloat(self, section, option): try: return self.p.getfloat(section, option) except: return tryInt(self.p.get(section, option)) def getUnicode(self, section, option): value = self.p.get(section, option).decode('unicode_escape') return toUnicode(value).strip() def getValues(self): values = {} for section in self.sections(): values[section] = {} for option in self.p.items(section): (option_name, option_value) = option values[section][option_name] = self.get(option_name, section) return values def save(self): with open(self.file, 'wb') as configfile: self.p.write(configfile) self.log.debug('Saved settings') def addSection(self, section): if not self.p.has_section(section): self.p.add_section(section) def setDefault(self, section, option, value): if not self.p.has_option(section, option): self.p.set(section, option, value) def setType(self, section, option, type): if not self.types.get(section): self.types[section] = {} self.types[section][option] = type def addOptions(self, section_name, options): if not self.options.get(section_name): self.options[section_name] = options else: self.options[section_name] = mergeDicts(self.options[section_name], options) def getOptions(self): return self.options def view(self): return { 'options': self.getOptions(), 'values': self.getValues() } def saveView(self, **kwargs): section = kwargs.get('section') option = kwargs.get('name') value = kwargs.get('value') # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True) self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single = True) return { 'success': True, } def getProperty(self, identifier): from couchpotato import get_session db = get_session() prop = None try: propert = db.query(Properties).filter_by(identifier = identifier).first() prop = propert.value except: pass return prop def setProperty(self, identifier, value = ''): from couchpotato import get_session db = get_session() p = db.query(Properties).filter_by(identifier = identifier).first() if not p: p = Properties() db.add(p) p.identifier = identifier p.value = toUnicode(value) db.commit()
from apscheduler.scheduler import Scheduler as Sched from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin import logging log = CPLog(__name__) class Scheduler(Plugin): crons = {} intervals = {} started = False def __init__(self): addEvent("schedule.cron", self.cron) addEvent("schedule.interval", self.interval) addEvent("schedule.start", self.start) addEvent("schedule.restart", self.start) addEvent("app.load", self.start) self.sched = Sched(misfire_grace_time=60) def remove(self, identifier): for type in ["interval", "cron"]: try: self.sched.unschedule_job(getattr(self, type)[identifier]["job"]) log.debug("%s unscheduled %s", (type.capitalize(), identifier))
from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import File, Movie from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info from subliminal.videos import Video import enzyme import os import re import time import traceback log = CPLog(__name__) class Scanner(Plugin): minimal_filesize = { 'media': 314572800, # 300MB 'trailer': 1048576, # 1MB } ignored_in_path = [ 'extracting', '_unpack', '_failed_', '_unknown_', '_exists_', '_failed_remove_', '_failed_rename_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo' ] #unpacking, smb-crap, hidden files ignore_names = [ 'extract', 'extracting', 'extracted', 'movie', 'movies', 'film',
from base64 import standard_b64encode from couchpotato.core.downloaders.base import Downloader, StatusList from couchpotato.core.helpers.encoding import ss from couchpotato.core.helpers.variable import tryInt, md5 from couchpotato.core.logger import CPLog from datetime import timedelta import re import shutil import socket import traceback import xmlrpclib log = CPLog(__name__) class NZBGet(Downloader): type = ['nzb'] url = 'http://%(username)s:%(password)s@%(host)s/xmlrpc' def download(self, data = {}, movie = {}, filedata = None): if not filedata: log.error('Unable to get NZB file: %s', traceback.format_exc()) return False log.info('Sending "%s" to NZBGet.', data.get('name')) url = self.url % {'host': self.conf('host'), 'username': self.conf('username'), 'password': self.conf('password')} nzb_name = ss('%s.nzb' % self.createNzbName(data, movie))
#!/usr/bin/env python from os.path import dirname from signal import signal, SIGTERM import os import subprocess import sys # Root path base_path = dirname(os.path.abspath(__file__)) # Insert local directories into path sys.path.insert(0, os.path.join(base_path, 'libs')) from couchpotato.core.logger import CPLog log = CPLog(__name__) # Get options via arg from couchpotato.runner import getOptions from couchpotato.core.helpers.variable import getDataDir options = getOptions(base_path, sys.argv[1:]) data_dir = getDataDir() def start(): try: args = [sys.executable] + [os.path.join(base_path, __file__)] + sys.argv[1:] new_environ = os.environ.copy() new_environ['cp_main'] = 'true' if os.name == 'nt': for key, value in new_environ.iteritems():
def runCouchPotato(options, base_path, args, data_dir=None, log_dir=None, Env=None, desktop=None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ("ANSI_X3.4-1968", "US-ASCII", "ASCII"): encoding = "UTF-8" Env.set("encoding", encoding) # Do db stuff db_path = sp(os.path.join(data_dir, "database")) old_db_path = os.path.join(data_dir, "couchpotato.db") # Remove database folder if both exists if os.path.isdir(db_path) and os.path.isfile(old_db_path): db = SuperThreadSafeDatabase(db_path) db.open() db.destroy() # Check if database exists db = SuperThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups backup_path = sp(os.path.join(data_dir, "db_backup")) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): # Only consider files being a direct child of the backup_path if root == backup_path: for backup_file in sorted(files): ints = re.findall("\d+", backup_file) # Delete non zip files if len(ints) != 1: try: os.remove(os.path.join(root, backup_file)) except: pass else: existing_backups.append((int(ints[0]), backup_file)) else: # Delete stray directories. shutil.rmtree(root) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: os.remove(os.path.join(backup_path, eb[1])) # Create new backup new_backup = sp(os.path.join(backup_path, "%s.tar.gz" % int(time.time()))) zipf = tarfile.open(new_backup, "w:gz") for root, dirs, files in os.walk(db_path): for zfilename in files: zipf.add( os.path.join(root, zfilename), arcname="database/%s" % os.path.join(root[len(db_path) + 1 :], zfilename), ) zipf.close() # Open last db.open() else: db.create() # Force creation of cachedir log_dir = sp(log_dir) cache_dir = sp(os.path.join(data_dir, "cache")) python_cache = sp(os.path.join(cache_dir, "python")) if not os.path.exists(cache_dir): os.mkdir(cache_dir) if not os.path.exists(python_cache): os.mkdir(python_cache) session = requests.Session() session.max_redirects = 5 # Register environment settings Env.set("app_dir", sp(base_path)) Env.set("data_dir", sp(data_dir)) Env.set("log_path", sp(os.path.join(log_dir, "CouchPotato.log"))) Env.set("db", db) Env.set("http_opener", session) Env.set("cache_dir", cache_dir) Env.set("cache", FileSystemCache(python_cache)) Env.set("console_log", options.console_log) Env.set("quiet", options.quiet) Env.set("desktop", desktop) Env.set("daemonized", options.daemon) Env.set("args", args) Env.set("options", options) # Determine debug debug = options.debug or Env.setting("debug", default=False, type="bool") Env.set("debug", debug) # Development development = Env.setting("development", default=False, type="bool") Env.set("dev", development) # Disable logging for some modules for logger_name in ["enzyme", "guessit", "subliminal", "apscheduler", "tornado", "requests"]: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ["gntp"]: logging.getLogger(logger_name).setLevel(logging.WARNING) # Disable SSL warning disable_warnings() # Use reloader reloader = debug is True and development and not Env.get("desktop") and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%m-%d %H:%M:%S") level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, "INFO") # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get("log_path"), "a", 500000, 10, encoding=Env.get("encoding")) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors # noinspection PyUnresolvedReferences import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug("Started with options %s", options) # Check soft-chroot dir exists: try: # Load Soft-Chroot soft_chroot = Env.get("softchroot") soft_chroot_dir = Env.setting("soft_chroot", section="core", default=None, type="unicode") soft_chroot.initialize(soft_chroot_dir) except SoftChrootInitError as exc: log.error(exc) return except: log.error("Unable to check whether SOFT-CHROOT is defined") return # Check available space try: total_space, available_space = getFreeSpace(data_dir) if available_space < 100: log.error( "Shutting down as CP needs some space to work. You'll get corrupted data otherwise. Only %sMB left", available_space, ) return except: log.error("Failed getting diskspace: %s", traceback.format_exc()) def customwarn(message, category, filename, lineno, file=None, line=None): log.warning("%s %s %s line:%s", (category, message, filename, lineno)) warnings.showwarning = customwarn # Create app from couchpotato import WebHandler web_base = ("/" + Env.setting("url_base").lstrip("/") + "/") if Env.setting("url_base") else "/" Env.set("web_base", web_base) api_key = Env.setting("api_key") if not api_key: api_key = uuid4().hex Env.setting("api_key", value=api_key) api_base = r"%sapi/%s/" % (web_base, api_key) Env.set("api_base", api_base) # Basic config host = Env.setting("host", default="0.0.0.0") host6 = Env.setting("host6", default="::") config = { "use_reloader": reloader, "port": tryInt(Env.setting("port", default=5050)), "host": host if host and len(host) > 0 else "0.0.0.0", "host6": host6 if host6 and len(host6) > 0 else "::", "ssl_cert": Env.setting("ssl_cert", default=None), "ssl_key": Env.setting("ssl_key", default=None), } # Load the app application = Application( [], log_function=lambda x: None, debug=config["use_reloader"], gzip=True, cookie_secret=api_key, login_url="%slogin/" % web_base, ) Env.set("app", application) # Request handlers application.add_handlers( ".*$", [ (r"%snonblock/(.*)(/?)" % api_base, NonBlockHandler), # API handlers (r"%s(.*)(/?)" % api_base, ApiHandler), # Main API handler (r"%sgetkey(/?)" % web_base, KeyHandler), # Get API key (r"%s" % api_base, RedirectHandler, {"url": web_base + "docs/"}), # API docs # Login handlers (r"%slogin(/?)" % web_base, LoginHandler), (r"%slogout(/?)" % web_base, LogoutHandler), # Catch all webhandlers (r"%s(.*)(/?)" % web_base, WebHandler), (r"(.*)", WebHandler), ], ) # Static paths static_path = "%sstatic/" % web_base for dir_name in ["fonts", "images", "scripts", "style"]: application.add_handlers( ".*$", [ ( "%s%s/(.*)" % (static_path, dir_name), StaticFileHandler, {"path": sp(os.path.join(base_path, "couchpotato", "static", dir_name))}, ) ], ) Env.set("static_path", static_path) # Load configs & plugins loader = Env.get("loader") loader.preload(root=sp(base_path)) loader.run() # Fill database with needed stuff fireEvent("database.setup") if not db_exists: fireEvent("app.initialize", in_order=True) fireEvent("app.migrate") # Go go go! from tornado.ioloop import IOLoop from tornado.autoreload import add_reload_hook loop = IOLoop.current() # Reload hook def reload_hook(): fireEvent("app.shutdown") add_reload_hook(reload_hook) # Some logging and fire load event try: log.info("Starting server on port %(port)s", config) except: pass fireEventAsync("app.load") ssl_options = None if config["ssl_cert"] and config["ssl_key"]: ssl_options = {"certfile": config["ssl_cert"], "keyfile": config["ssl_key"]} server = HTTPServer(application, no_keep_alive=True, ssl_options=ssl_options) try_restart = True restart_tries = 5 while try_restart: try: if config["host"].startswith("unix:"): server.add_socket(bind_unix_socket(config["host"][5:])) else: server.listen(config["port"], config["host"]) if Env.setting("ipv6", default=False): try: server.listen(config["port"], config["host6"]) except: log.info2("Tried to bind to IPV6 but failed") loop.start() server.close_all_connections() server.stop() loop.close(all_fds=True) except Exception as e: log.error("Failed starting: %s", traceback.format_exc()) try: nr, msg = e if nr == 48: log.info( "Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds", (config.get("port"), restart_tries), ) time.sleep(1) restart_tries -= 1 if restart_tries > 0: continue else: return except ValueError: return except: pass raise try_restart = False
from couchpotato.core.downloaders.base import Downloader from couchpotato.core.helpers.encoding import isInt from couchpotato.core.logger import CPLog import httplib import json import urllib import urllib2 log = CPLog(__name__) class Synology(Downloader): type = ['torrent_magnet'] log = CPLog(__name__) def download(self, data, movie, manual = False, filedata = None): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return log.error('Sending "%s" (%s) to Synology.', (data.get('name'), data.get('type'))) # Load host from config and split out port. host = self.conf('host').split(':') if not isInt(host[1]): log.error('Config properties are not filled in correctly, port is missing.') return False if data.get('type') == 'torrent': log.error('Can\'t add binary torrent file')
from axl.axel import Event from couchpotato.core.helpers.variable import mergeDicts, natcmp from couchpotato.core.logger import CPLog import threading import traceback log = CPLog(__name__) events = {} def runHandler(name, handler, *args, **kwargs): try: return handler(*args, **kwargs) except: from couchpotato.environment import Env log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all())) def addEvent(name, handler, priority = 100): if events.get(name): e = events[name] else: e = events[name] = Event(name = name, threads = 10, exc_info = True, traceback = True, lock = threading.RLock()) def createHandle(*args, **kwargs): try: parent = handler.im_self bc = hasattr(parent, 'beforeCall') if bc: parent.beforeCall(handler) h = runHandler(name, handler, *args, **kwargs) ac = hasattr(parent, 'afterCall')
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None): try: locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): encoding = 'UTF-8' Env.set('encoding', encoding) # Do db stuff db_path = sp(os.path.join(data_dir, 'database')) old_db_path = os.path.join(data_dir, 'couchpotato.db') # Remove database folder if both exists if os.path.isdir(db_path) and os.path.isfile(old_db_path): db = SuperThreadSafeDatabase(db_path) db.open() db.destroy() # Check if database exists db = SuperThreadSafeDatabase(db_path) db_exists = db.exists() if db_exists: # Backup before start and cleanup old backups backup_path = sp(os.path.join(data_dir, 'db_backup')) backup_count = 5 existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) for root, dirs, files in os.walk(backup_path): # Only consider files being a direct child of the backup_path if root == backup_path: for backup_file in sorted(files): ints = re.findall('\d+', backup_file) # Delete non zip files if len(ints) != 1: try: os.remove(os.path.join(root, backup_file)) except: pass else: existing_backups.append((int(ints[0]), backup_file)) else: # Delete stray directories. shutil.rmtree(root) # Remove all but the last 5 for eb in existing_backups[:-backup_count]: os.remove(os.path.join(backup_path, eb[1])) # Create new backup new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') for root, dirs, files in os.walk(db_path): for zfilename in files: zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename)) zipf.close() # Open last db.open() else: db.create() # Force creation of cachedir log_dir = sp(log_dir) cache_dir = sp(os.path.join(data_dir, 'cache')) python_cache = sp(os.path.join(cache_dir, 'python')) if not os.path.exists(cache_dir): os.mkdir(cache_dir) if not os.path.exists(python_cache): os.mkdir(python_cache) session = requests.Session() session.max_redirects = 5 # Register environment settings Env.set('app_dir', sp(base_path)) Env.set('data_dir', sp(data_dir)) Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log'))) Env.set('db', db) Env.set('http_opener', session) Env.set('cache_dir', cache_dir) Env.set('cache', FileSystemCache(python_cache)) Env.set('console_log', options.console_log) Env.set('quiet', options.quiet) Env.set('desktop', desktop) Env.set('daemonized', options.daemon) Env.set('args', args) Env.set('options', options) # Determine debug debug = options.debug or Env.setting('debug', default = False, type = 'bool') Env.set('debug', debug) # Development development = Env.setting('development', default = False, type = 'bool') Env.set('dev', development) # Disable logging for some modules for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']: logging.getLogger(logger_name).setLevel(logging.ERROR) for logger_name in ['gntp']: logging.getLogger(logger_name).setLevel(logging.WARNING) # Disable SSL warning disable_warnings() # Use reloader reloader = debug is True and development and not Env.get('desktop') and not options.daemon # Logger logger = logging.getLogger() formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S') level = logging.DEBUG if debug else logging.INFO logger.setLevel(level) logging.addLevelName(19, 'INFO') # To screen if (debug or options.console_log) and not options.quiet and not options.daemon: hdlr = logging.StreamHandler(sys.stderr) hdlr.setFormatter(formatter) logger.addHandler(hdlr) # To file hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding')) hdlr2.setFormatter(formatter) logger.addHandler(hdlr2) # Start logging & enable colors # noinspection PyUnresolvedReferences import color_logs from couchpotato.core.logger import CPLog log = CPLog(__name__) log.debug('Started with options %s', options) # Check soft-chroot dir exists: try: # Load Soft-Chroot soft_chroot = Env.get('softchroot') soft_chroot_dir = Env.setting('soft_chroot', section = 'core', default = None, type='unicode' ) soft_chroot.initialize(soft_chroot_dir) except SoftChrootInitError as exc: log.error(exc) return except: log.error('Unable to check whether SOFT-CHROOT is defined') return # Check available space try: total_space, available_space = getFreeSpace(data_dir) if available_space < 100: log.error('Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space) return except: log.error('Failed getting diskspace: %s', traceback.format_exc()) def customwarn(message, category, filename, lineno, file = None, line = None): log.warning('%s %s %s line:%s', (category, message, filename, lineno)) warnings.showwarning = customwarn # Create app from couchpotato import WebHandler web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/' Env.set('web_base', web_base) api_key = Env.setting('api_key') if not api_key: api_key = uuid4().hex Env.setting('api_key', value = api_key) api_base = r'%sapi/%s/' % (web_base, api_key) Env.set('api_base', api_base) # Basic config host = Env.setting('host', default = '0.0.0.0') host6 = Env.setting('host6', default = '::') config = { 'use_reloader': reloader, 'port': tryInt(Env.setting('port', default = 5050)), 'host': host if host and len(host) > 0 else '0.0.0.0', 'host6': host6 if host6 and len(host6) > 0 else '::', 'ssl_cert': Env.setting('ssl_cert', default = None), 'ssl_key': Env.setting('ssl_key', default = None), } # Load the app application = Application( [], log_function = lambda x: None, debug = config['use_reloader'], gzip = True, cookie_secret = api_key, login_url = '%slogin/' % web_base, ) Env.set('app', application) # Request handlers application.add_handlers(".*$", [ (r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler), # API handlers (r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler (r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key (r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs # Login handlers (r'%slogin(/?)' % web_base, LoginHandler), (r'%slogout(/?)' % web_base, LogoutHandler), # Catch all webhandlers (r'%s(.*)(/?)' % web_base, WebHandler), (r'(.*)', WebHandler), ]) # Static paths static_path = '%sstatic/' % web_base for dir_name in ['fonts', 'images', 'scripts', 'style']: application.add_handlers(".*$", [ ('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))}) ]) Env.set('static_path', static_path) # Load configs & plugins loader = Env.get('loader') loader.preload(root = sp(base_path)) loader.run() # Fill database with needed stuff fireEvent('database.setup') if not db_exists: fireEvent('app.initialize', in_order = True) fireEvent('app.migrate') # Go go go! from tornado.ioloop import IOLoop from tornado.autoreload import add_reload_hook loop = IOLoop.current() # Reload hook def reload_hook(): fireEvent('app.shutdown') add_reload_hook(reload_hook) # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass fireEventAsync('app.load') ssl_options = None if config['ssl_cert'] and config['ssl_key']: ssl_options = { 'certfile': config['ssl_cert'], 'keyfile': config['ssl_key'], } server = HTTPServer(application, no_keep_alive = True, ssl_options = ssl_options) run_tries = 5 assert run_tries > 0 while run_tries > 0: run_tries -= 1 try: server.listen(config['port'], config['host']) if Env.setting('ipv6', default = False): try: server.listen(config['port'], config['host6']) except: log.info2('Tried to bind to IPV6 but failed') loop.start() # on shutting down without exception server.close_all_connections() server.stop() loop.close(all_fds = True) except SocketError as e: # here we will handle just two errors : # errno.EADDRINUSE = 98 - address in use # errno.ELNRNG = 48 - port in use (strange, but I took it from old code) # TODO : check, that value 48 still actual if e.errno in [ errno.EADDRINUSE, errno.ELNRNG ]: if run_tries > 0: log.warning('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), run_tries)) time.sleep(2) continue # not ugly error message log.error('Failed starting: Port (%s) needed for CouchPotato is already in use', (config['port'])) # we will not raise this exception again, because no additional actions are required: return log.error('Failed starting: %s', traceback.format_exc()) raise except Exception as e: log.error('Failed starting: %s', traceback.format_exc()) raise pass # while run_tries>0