def getCache(self, cache_key, url=None, **kwargs):
use_cache = not len(kwargs.get("data", {})) > 0 and not kwargs.get("files")
if use_cache:
cache_key_md5 = md5(cache_key)
cache = Env.get("cache").get(cache_key_md5)
if cache:
if not Env.get("dev"):
log.debug("Getting cache %s", cache_key)
return cache
if url:
try:
cache_timeout = 300
if "cache_timeout" in kwargs:
cache_timeout = kwargs.get("cache_timeout")
del kwargs["cache_timeout"]
data = self.urlopen(url, **kwargs)
if data and cache_timeout > 0 and use_cache:
self.setCache(cache_key, data, timeout=cache_timeout)
return data
except:
if not kwargs.get("show_error", True):
raise
log.debug("Failed getting cache: %s", (traceback.format_exc(0)))
return ""
def replaceWith(self, path):
app_dir = Env.get('app_dir')
# Get list of files we want to overwrite
self.deletePyc(only_excess = False)
existing_files = []
for root, subfiles, filenames in os.walk(app_dir):
for filename in filenames:
existing_files.append(os.path.join(root, filename))
for root, subfiles, filenames in os.walk(path):
for filename in filenames:
fromfile = os.path.join(root, filename)
tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, ''))
if not Env.get('dev'):
try:
os.remove(tofile)
except:
pass
try:
os.renames(fromfile, tofile)
try:
existing_files.remove(tofile)
except ValueError:
pass
except Exception, e:
log.error('Failed overwriting file: %s' % e)
def __init__(self):
if Env.get('desktop'):
self.updater = DesktopUpdater()
elif os.path.isdir(os.path.join(Env.get('app_dir'), '.git')):
self.updater = GitUpdater(self.conf('git_command', default = 'git'))
else:
self.updater = SourceUpdater()
fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6)
addEvent('app.load', self.autoUpdate)
addEvent('updater.info', self.info)
addApiView('updater.info', self.getInfo, docs = {
'desc': 'Get updater information',
'return': {
'type': 'object',
'example': """{
'last_check': "last checked for update",
'update_version': "available update version or empty",
'version': current_cp_version
}"""}
})
addApiView('updater.update', self.doUpdateView)
addApiView('updater.check', self.checkView, docs = {
'desc': 'Check for available update',
'return': {'type': 'see updater.info'}
})
def getCache(self, cache_key, url = None, **kwargs):
cache_key = simplifyString(cache_key)
cache = Env.get('cache').get(cache_key)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
return cache
if url:
try:
cache_timeout = 300
if kwargs.get('cache_timeout'):
cache_timeout = kwargs.get('cache_timeout')
del kwargs['cache_timeout']
opener = None
if kwargs.get('opener'):
opener = kwargs.get('opener')
del kwargs['opener']
if opener:
log.info('Opening url: %s', url)
f = opener.open(url)
data = f.read()
f.close()
else:
data = self.urlopen(url, **kwargs)
if data:
self.setCache(cache_key, data, timeout = cache_timeout)
return data
except:
pass
def registerStatic(self, plugin_file, add_to_head=True):
# Register plugin path
self.plugin_path = os.path.dirname(plugin_file)
static_folder = toUnicode(os.path.join(self.plugin_path, "static"))
if not os.path.isdir(static_folder):
return
# Get plugin_name from PluginName
s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__)
class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
# View path
path = "static/plugin/%s/" % class_name
# Add handler to Tornado
Env.get("app").add_handlers(
".*$", [(Env.get("web_base") + path + "(.*)", StaticFileHandler, {"path": static_folder})]
)
# Register for HTML <HEAD>
if add_to_head:
for f in glob.glob(os.path.join(self.plugin_path, "static", "*")):
ext = getExt(f)
if ext in ["js", "css"]:
fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f), f)
def doUpdate(self):
try:
url = 'https://github.com/%s/%s/tarball/%s' % (self.repo_user, self.repo_name, self.branch)
destination = os.path.join(Env.get('cache_dir'), self.update_version.get('hash') + '.tar.gz')
extracted_path = os.path.join(Env.get('cache_dir'), 'temp_updater')
destination = fireEvent('file.download', url = url, dest = destination, single = True)
# Cleanup leftover from last time
if os.path.isdir(extracted_path):
self.removeDir(extracted_path)
self.makeDir(extracted_path)
# Extract
tar = tarfile.open(destination)
tar.extractall(path = extracted_path)
tar.close()
os.remove(destination)
if self.replaceWith(os.path.join(extracted_path, os.listdir(extracted_path)[0])):
self.removeDir(extracted_path)
# Write update version to file
self.createFile(self.version_file, json.dumps(self.update_version))
return True
except:
log.error('Failed updating: %s', traceback.format_exc())
self.update_failed = True
return False
def registerStatic(self, plugin_file, add_to_head = True):
# Register plugin path
self.plugin_path = os.path.dirname(plugin_file)
static_folder = toUnicode(os.path.join(self.plugin_path, 'static'))
if not os.path.isdir(static_folder):
return
# Get plugin_name from PluginName
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
# View path
path = 'static/plugin/%s/' % class_name
# Add handler to Tornado
Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})])
# Register for HTML <HEAD>
if add_to_head:
for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')):
ext = getExt(f)
if ext in ['js', 'css']:
fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f)
def __init__(self):
if Env.get("desktop"):
self.updater = DesktopUpdater()
elif os.path.isdir(os.path.join(Env.get("app_dir"), ".git")):
self.updater = GitUpdater(self.conf("git_command", default="git"))
else:
self.updater = SourceUpdater()
addEvent("app.load", self.setCrons)
addEvent("updater.info", self.info)
addApiView(
"updater.info",
self.getInfo,
docs={
"desc": "Get updater information",
"return": {
"type": "object",
"example": """{
'last_check': "last checked for update",
'update_version': "available update version or empty",
'version': current_cp_version
}""",
},
},
)
addApiView("updater.update", self.doUpdateView)
addApiView(
"updater.check",
self.checkView,
docs={"desc": "Check for available update", "return": {"type": "see updater.info"}},
)
addEvent("setting.save.updater.enabled.after", self.setCrons)
def getCache(self, cache_key, url=None, **kwargs):
cache_key_md5 = md5(cache_key)
cache = Env.get("cache").get(cache_key_md5)
if cache:
if not Env.get("dev"):
log.debug("Getting cache %s", cache_key)
return cache
if url:
try:
cache_timeout = 300
if kwargs.get("cache_timeout"):
cache_timeout = kwargs.get("cache_timeout")
del kwargs["cache_timeout"]
data = self.urlopen(url, **kwargs)
if data:
self.setCache(cache_key, data, timeout=cache_timeout)
return data
except:
if not kwargs.get("show_error", True):
raise
return ""
def getCache(self, cache_key, url = None, **kwargs):
use_cache = not len(kwargs.get('data', {})) > 0 and not kwargs.get('files')
if use_cache:
cache_key_md5 = md5(cache_key)
cache = Env.get('cache').get(cache_key_md5)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
return cache
if url:
try:
cache_timeout = 300
if 'cache_timeout' in kwargs:
cache_timeout = kwargs.get('cache_timeout')
del kwargs['cache_timeout']
data = self.urlopen(url, **kwargs)
if data and cache_timeout > 0 and use_cache:
self.setCache(cache_key, data, timeout = cache_timeout)
return data
except:
if not kwargs.get('show_error', True):
raise
log.debug('Failed getting cache: %s', (traceback.format_exc(0)))
return ''
def getCache(self, cache_key, url = None, **kwargs):
cache_key = md5(ss(cache_key))
cache = Env.get('cache').get(cache_key)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
return cache
if url:
try:
cache_timeout = 300
if kwargs.get('cache_timeout'):
cache_timeout = kwargs.get('cache_timeout')
del kwargs['cache_timeout']
data = self.urlopen(url, **kwargs)
if data:
self.setCache(cache_key, data, timeout = cache_timeout)
return data
except:
if not kwargs.get('show_error', True):
raise
return ''
def getUserScript(self, script_route, **kwargs):
klass = self
class UserscriptHandler(RequestHandler):
def get(self, random, route):
bookmarklet_host = Env.setting('bookmarklet_host')
loc = bookmarklet_host if bookmarklet_host else "{0}://{1}".format(self.request.protocol, self.request.headers.get('X-Forwarded-Host') or self.request.headers.get('host'))
params = {
'includes': fireEvent('userscript.get_includes', merge = True),
'excludes': fireEvent('userscript.get_excludes', merge = True),
'version': klass.getVersion(),
'api': '%suserscript/' % Env.get('api_base'),
'host': loc,
}
script = klass.renderTemplate(__file__, 'template.js_tmpl', **params)
klass.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script)
self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js')
Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), script_route), UserscriptHandler)])
def __init__(self):
if Env.get('desktop'):
self.updater = DesktopUpdater()
elif os.path.isdir(os.path.join(Env.get('app_dir'), '.git')):
self.updater = GitUpdater(self.conf('git_command', default = 'git'))
else:
self.updater = SourceUpdater()
addEvent('app.load', self.logVersion, priority = 10000)
addEvent('app.load', self.setCrons)
addEvent('updater.info', self.info)
addApiView('updater.info', self.info, docs = {
'desc': 'Get updater information',
'return': {
'type': 'object',
'example': """{
'last_check': "last checked for update",
'update_version': "available update version or empty",
'version': current_cp_version
}"""}
})
addApiView('updater.update', self.doUpdateView)
addApiView('updater.check', self.checkView, docs = {
'desc': 'Check for available update',
'return': {'type': 'see updater.info'}
})
addEvent('setting.save.updater.enabled.after', self.setCrons)
def checkDataDir(self):
if Env.get("app_dir") in Env.get("data_dir"):
log.error(
"You should NOT use your CouchPotato directory to save your settings in. Files will get overwritten or be deleted."
)
return True
def getCredentials(self, **kwargs):
try:
oauth_token = kwargs.get('oauth')
except:
return 'redirect', Env.get('web_base') + 'settings/downloaders/'
log.debug('oauth_token is: %s', oauth_token)
self.conf('oauth_token', value = oauth_token);
return 'redirect', Env.get('web_base') + 'settings/downloaders/'
def search(self):
params = getParams()
cache_key = "%s/%s" % (__name__, urlencode(params))
movies = Env.get("cache").get(cache_key)
if not movies:
movies = fireEvent("movie.search", q=params.get("q"), merge=True)
Env.get("cache").set(cache_key, movies)
return jsonified({"success": True, "empty": len(movies) == 0 if movies else 0, "movies": movies})
def ss(original, *args):
u_original = toUnicode(original, *args)
try:
from couchpotato.environment import Env
return u_original.encode(Env.get('encoding'))
except Exception as e:
log.debug('Failed ss encoding char, force UTF8: %s', e)
try:
return u_original.encode(Env.get('encoding'), 'replace')
except:
return u_original.encode('utf-8', 'replace')
def getCache(self, cache_key, url = None, timeout = 300):
cache = Env.get('cache').get(cache_key)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s' % cache_key)
return cache
if url:
try:
data = self.urlopen(url)
self.setCache(cache_key, data, timeout = timeout)
return data
except:
pass
def page_not_found(rh):
index_url = Env.get('web_base')
url = rh.request.uri[len(index_url):]
if url[:3] != 'api':
r = index_url + '#' + url.lstrip('/')
rh.redirect(r)
else:
if not Env.get('dev'):
time.sleep(0.1)
rh.set_status(404)
rh.write('Wrong API key used')
def get(self, random, route):
params = {
'includes': fireEvent('userscript.get_includes', merge = True),
'excludes': fireEvent('userscript.get_excludes', merge = True),
'version': klass.getVersion(),
'api': '%suserscript/' % Env.get('api_base'),
'host': '%s://%s' % (self.request.protocol, self.request.host),
}
script = klass.renderTemplate(__file__, 'template.js', **params)
klass.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script)
self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js')
def ss(original, *args):
u_original = toUnicode(original, *args)
try:
from couchpotato.environment import Env
return u_original.encode(Env.get("encoding"))
except Exception as e:
log.debug("Failed ss encoding char, force UTF8: %s", e)
return u_original.encode("utf8")
try:
return u_original.encode(Env.get("encoding"), "replace")
except:
return u_original.encode("utf-8", "replace")
def minify(self):
# Create cache dir
cache = Env.get('cache_dir')
parent_dir = os.path.join(cache, 'minified')
self.makeDir(parent_dir)
Env.get('app').add_handlers(".*$", [(Env.get('web_base') + 'minified/(.*)', StaticFileHandler, {'path': parent_dir})])
for file_type in ['style', 'script']:
ext = 'js' if file_type is 'script' else 'css'
positions = self.paths.get(file_type, {})
for position in positions:
files = positions.get(position)
self._minify(file_type, files, position, position + '.' + ext)
def search(self):
q = getParam("q")
cache_key = u"%s/%s" % (__name__, simplifyString(q))
movies = Env.get("cache").get(cache_key)
if not movies:
if getImdb(q):
movies = [fireEvent("movie.info", identifier=q, merge=True)]
else:
movies = fireEvent("movie.search", q=q, merge=True)
Env.get("cache").set(cache_key, movies)
return jsonified({"success": True, "empty": len(movies) == 0 if movies else 0, "movies": movies})
def search(self):
params = getParams()
cache_key = '%s/%s' % (__name__, urlencode(params))
movies = Env.get('cache').get(cache_key)
if not movies:
movies = fireEvent('movie.search', q = params.get('q'), merge = True)
Env.get('cache').set(cache_key, movies)
return jsonified({
'success': True,
'empty': len(movies) == 0 if movies else 0,
'movies': movies,
})
def getUserScript(self, filename = ''):
params = {
'includes': fireEvent('userscript.get_includes', merge = True),
'excludes': fireEvent('userscript.get_excludes', merge = True),
'version': self.getVersion(),
'api': '%suserscript/' % url_for('api.index').lstrip('/'),
'host': request.host_url,
}
script = self.renderTemplate(__file__, 'template.js', **params)
self.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script)
from flask.helpers import send_from_directory
return send_from_directory(Env.get('cache_dir'), 'couchpotato.user.js')
def fromOld(self):
if request.method != 'POST':
return self.renderTemplate(__file__, 'form.html', url_for = url_for)
file = request.files['old_db']
uploaded_file = os.path.join(Env.get('cache_dir'), 'v1_database.db')
if os.path.isfile(uploaded_file):
os.remove(uploaded_file)
file.save(uploaded_file)
try:
import sqlite3
conn = sqlite3.connect(uploaded_file)
wanted = []
t = ('want',)
cur = conn.execute('SELECT status, imdb FROM Movie WHERE status=?', t)
for row in cur:
status, imdb = row
if getImdb(imdb):
wanted.append(imdb)
conn.close()
wanted = set(wanted)
for imdb in wanted:
fireEventAsync('movie.add', {'identifier': imdb}, search_after = False)
message = 'Successfully imported %s movie(s)' % len(wanted)
except Exception, e:
message = 'Failed: %s' % e
def check(self):
if self.update_version:
return True
log.info('Checking for new version on github for %s', self.repo_name)
if not Env.get('dev'):
self.repo.fetch()
current_branch = self.repo.getCurrentBranch().name
for branch in self.repo.getRemoteByName('origin').getBranches():
if current_branch == branch.name:
local = self.repo.getHead()
remote = branch.getHead()
log.info('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8]))
if local.getDate() < remote.getDate():
self.update_version = {
'hash': remote.hash[:8],
'date': remote.getDate(),
}
return True
self.last_check = time.time()
return False
def showCacheFile(self, filename = ''):
cache_dir = Env.get('cache_dir')
filename = os.path.basename(filename)
from flask.helpers import send_from_directory
return send_from_directory(cache_dir, filename)
def __init__(self):
fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2)
addEvent('manage.update', self.updateLibrary)
addEvent('manage.diskspace', self.getDiskSpace)
# Add files after renaming
def after_rename(message = None, group = None):
if not group: group = {}
return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
addEvent('renamer.after', after_rename, priority = 110)
addApiView('manage.update', self.updateLibraryView, docs = {
'desc': 'Update the library by scanning for new movies',
'params': {
'full': {'desc': 'Do a full update or just recently changed/added movies.'},
}
})
addApiView('manage.progress', self.getProgress, docs = {
'desc': 'Get the progress of current manage update',
'return': {'type': 'object', 'example': """{
'progress': False || object, total & to_go,
}"""},
})
if not Env.get('dev') and self.conf('startup_scan'):
addEvent('app.load', self.updateLibraryQuick)
def safeMessage(self, msg, replace_tuple = ()):
from couchpotato.environment import Env
from couchpotato.core.helpers.encoding import ss
msg = ss(msg)
try:
msg = msg % replace_tuple
except:
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) for x in list(replace_tuple)])
else:
msg = msg % ss(replace_tuple)
except:
self.logger.error(u'Failed encoding stuff to log: %s' % traceback.format_exc())
if not Env.get('dev'):
for replace in self.replace_private:
msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace, msg)
msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace, msg)
# Replace api key
try:
api_key = Env.setting('api_key')
if api_key:
msg = msg.replace(api_key, 'API_KEY')
except:
pass
return msg
def download(self, url='', dest=None, overwrite=False):
try:
file = urllib2.urlopen(url)
if not dest: # to Cache
dest = os.path.join(Env.get('cache_dir'),
'%s.%s' % (md5(url), getExt(url)))
if overwrite or not os.path.exists(dest):
log.debug('Writing file to: %s' % dest)
output = open(dest, 'wb')
output.write(file.read())
output.close()
else:
log.debug('File already exists: %s' % dest)
return dest
except Exception, e:
log.error('Unable to download file "%s": %s' % (url, e))
def download(self, data=None, media=None, filedata=None):
if not media: media = {}
if not data: data = {}
log.info('Sending "%s" to put.io', data.get('name'))
url = data.get('url')
client = pio.Client(self.conf('oauth_token'))
putioFolder = self.convertFolder(client, self.conf('folder'))
log.debug('putioFolder ID is %s', putioFolder)
# It might be possible to call getFromPutio from the renamer if we can then we don't need to do this.
# Note callback_host is NOT our address, it's the internet host that putio can call too
callbackurl = None
if self.conf('download'):
callbackurl = 'http://' + self.conf(
'callback_host') + '%sdownloader.putio.getfrom/' % Env.get(
'api_base'.strip('/'))
resp = client.Transfer.add_url(url,
callback_url=callbackurl,
parent_id=putioFolder)
log.debug('resp is %s', resp.id)
return self.downloadReturnId(resp.id)
def clear(self, **kwargs):
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
if not os.path.isfile(path):
continue
try:
# Create empty file for current logging
if x is 0:
self.createFile(path, '')
else:
os.remove(path)
except:
log.error('Couldn\'t delete file "%s": %s',
(path, traceback.format_exc()))
return {'success': True}
def searchLibrary(self):
# Get all active and online movies
db = get_session()
library = db.query(Library).all()
done_status = fireEvent('status.get', 'done', single = True)
for movie in library.movies:
for release in movie.releases:
# get releases and their movie files
if release.status_id is done_status.get('id'):
files = []
for file in release.files.filter(FileType.status.has(identifier = 'movie')).all():
files.append(file.path)
# get subtitles for those files
subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services)
def register(self):
if self.registered: return
try:
hostname = self.conf('hostname')
password = self.conf('password')
port = self.conf('port')
self.growl = notifier.GrowlNotifier(
applicationName=Env.get('appname'),
notifications=["Updates"],
defaultNotifications=["Updates"],
applicationIcon='%s/static/images/couch.png' %
fireEvent('app.api_url', single=True),
hostname=hostname if hostname else 'localhost',
password=password if password else None,
port=port if port else 23053)
self.growl.register()
self.registered = True
except:
log.error('Failed register of growl: %s', traceback.format_exc())
def partial(self, type = 'all', lines = 30, offset = 0, **kwargs):
total_lines = try_int(lines)
offset = try_int(offset)
log_lines = []
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
# Check see if the log exists
if not os.path.isfile(path):
break
f = open(path, 'r')
log_content = to_unicode(f.read())
raw_lines = self.toList(log_content)
raw_lines.reverse()
brk = False
for line in raw_lines:
if type == 'all' or line.get('type') == type.upper():
log_lines.append(line)
if len(log_lines) >= (total_lines + offset):
brk = True
break
if brk:
break
log_lines = log_lines[offset:]
log_lines.reverse()
return {
'success': True,
'log': log_lines,
}
def __init__(self):
super(CoreNotifier, self).__init__()
addEvent('notify', self.notify)
addEvent('notify.frontend', self.frontend)
addApiView('notification.markread', self.markAsRead, docs = {
'desc': 'Mark notifications as read',
'params': {
'ids': {'desc': 'Notification id you want to mark as read. All if ids is empty.', 'type': 'int (comma separated)'},
},
})
addApiView('notification.list', self.listView, docs = {
'desc': 'Get list of notifications',
'params': {
'limit_offset': {'desc': 'Limit and offset the notification list. Examples: "50" or "50,30"'},
},
'return': {'type': 'object', 'example': """{
'success': True,
'empty': bool, any notification returned or not,
'notifications': array, notifications found,
}"""}
})
addNonBlockApiView('notification.listener', (self.addListener, self.removeListener))
addApiView('notification.listener', self.listener)
fireEvent('schedule.interval', 'core.check_messages', self.checkMessages, hours = 12, single = True)
fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True)
addEvent('app.load', self.clean)
if not Env.get('dev'):
addEvent('app.load', self.checkMessages)
self.messages = []
self.listeners = []
self.m_lock = threading.Lock()
def saveView(self, **kwargs):
section = kwargs.get('section')
option = kwargs.get('name')
value = kwargs.get('value')
if not self.isOptionWritable(section, option):
self.log.warning('Option "%s.%s" isn\'t writable', (section, option))
return {
'success' : False,
}
from couchpotato.environment import Env
soft_chroot = Env.get('softchroot')
if self.getType(section, option) == 'directory':
value = soft_chroot.chroot2abs(value)
if self.getType(section, option) == 'directories':
import json
value = json.loads(value)
if not (value and isinstance(value, list)):
value = []
value = map(soft_chroot.chroot2abs, value)
value = self.directories_delimiter.join(value)
# See if a value handler is attached, use that as value
new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True)
self.set(section, option, (new_value if new_value else value).encode('unicode_escape'))
self.save()
# After save (for re-interval etc)
fireEvent('setting.save.%s.%s.after' % (section, option), single = True)
fireEvent('setting.save.%s.*.after' % section, single = True)
return {
'success': True
}
def _minify(self, file_type, files, position, out):
cache = Env.get('cache_dir')
out_name = 'minified_' + out
out = os.path.join(cache, out_name)
raw = []
for file_path in files:
f = open(file_path, 'r').read()
if file_type == 'script':
data = jsmin(f)
else:
data = cssmin(f)
data = data.replace('../images/', '../static/images/')
raw.append({
'file': file_path,
'date': int(os.path.getmtime(file_path)),
'data': data
})
# Combine all files together with some comments
data = ''
for r in raw:
data += self.comment.get(file_type) % (r.get('file'),
r.get('date'))
data += r.get('data') + '\n\n'
self.createFile(out, data.strip())
if not self.minified.get(file_type):
self.minified[file_type] = {}
if not self.minified[file_type].get(position):
self.minified[file_type][position] = []
minified_url = 'api/%s/file.cache/%s?%s' % (
Env.setting('api_key'), out_name, tryInt(os.path.getmtime(out)))
self.minified[file_type][position].append(minified_url)
def searchSingle(self, group):
if self.isDisabled(): return
try:
available_languages = sum(group['subtitle_language'].itervalues(),
[])
downloaded = []
files = [toUnicode(x) for x in group['files']['movie']]
log.debug('Searching for subtitles for: %s', files)
for lang in self.getLanguages():
if lang not in available_languages:
download = subliminal.download_subtitles(
files,
multi=True,
force=False,
languages=[lang],
services=self.services,
cache_dir=Env.get('cache_dir'))
for subtitle in download:
downloaded.extend(download[subtitle])
for d_sub in downloaded:
log.info('Found subtitle (%s): %s',
(d_sub.language.alpha2, files))
group['files']['subtitle'].append(d_sub.path)
group['subtitle_language'][d_sub.path] = [
d_sub.language.alpha2
]
return True
except:
log.error('Failed searching for subtitle: %s',
(traceback.format_exc()))
return False
def partial(self):
log_type = getParam('type', 'all')
total_lines = tryInt(getParam('lines', 30))
log_lines = []
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
# Check see if the log exists
if not os.path.isfile(path):
break
reversed_lines = []
f = open(path, 'r')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
brk = False
for line in reversed_lines:
if log_type == 'all' or '%s ' % log_type.upper() in line:
log_lines.append(line)
if len(log_lines) >= total_lines:
brk = True
break
if brk:
break
log_lines.reverse()
return jsonified({
'success': True,
'log': '[0m\n'.join(log_lines),
})
def __init__(self):
# Get options via arg
from couchpotato.runner import getOptions
self.options = getOptions(base_path, sys.argv[1:])
# Load settings
settings = Env.get('settings')
settings.setFile(self.options.config_file)
# Create data dir if needed
if self.options.data_dir:
self.data_dir = self.options.data_dir
else:
self.data_dir = os.path.expanduser(Env.setting('data_dir'))
if self.data_dir == '':
self.data_dir = getDataDir()
if not os.path.isdir(self.data_dir):
os.makedirs(self.data_dir)
# Create logging dir
self.log_dir = os.path.join(self.data_dir, 'logs')
if not os.path.isdir(self.log_dir):
os.mkdir(self.log_dir)
# Logging
from couchpotato.core.logger import CPLog
self.log = CPLog(__name__)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%H:%M:%S')
hdlr = handlers.RotatingFileHandler(
os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10)
hdlr.setLevel(logging.CRITICAL)
hdlr.setFormatter(formatter)
self.log.logger.addHandler(hdlr)
def __init__(self):
addApiView('app.shutdown',
self.shutdown,
docs={
'desc': 'Shutdown the app.',
'return': {
'type': 'string: shutdown'
}
})
addApiView('app.restart',
self.restart,
docs={
'desc': 'Restart the app.',
'return': {
'type': 'string: restart'
}
})
addApiView('app.available',
self.available,
docs={'desc': 'Check if app available.'})
addApiView('app.version',
self.versionView,
docs={'desc': 'Get version.'})
addEvent('app.shutdown', self.shutdown)
addEvent('app.restart', self.restart)
addEvent('app.load', self.launchBrowser, priority=1)
addEvent('app.base_url', self.createBaseUrl)
addEvent('app.api_url', self.createApiUrl)
addEvent('app.version', self.version)
addEvent('app.load', self.checkDataDir)
addEvent('setting.save.core.password', self.md5Password)
addEvent('setting.save.core.api_key', self.checkApikey)
# Make sure we can close-down with ctrl+c properly
if not Env.get('desktop'):
self.signalHandler()
def download(self,
url='',
dest=None,
overwrite=False,
urlopen_kwargs=None):
if not urlopen_kwargs: urlopen_kwargs = {}
if not dest: # to Cache
dest = os.path.join(Env.get('cache_dir'),
'%s.%s' % (md5(url), getExt(url)))
if not overwrite and os.path.isfile(dest):
return dest
try:
filedata = self.urlopen(url, **urlopen_kwargs)
except:
log.error('Failed downloading file %s: %s',
(url, traceback.format_exc()))
return False
self.createFile(dest, filedata, binary=True)
return dest
def deletePyc(self, only_excess = True):
for root, dirs, files in os.walk(ss(Env.get('app_dir'))):
pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
py_files = set(filter(lambda filename: filename.endswith('.py'), files))
excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
for excess_pyc_file in excess_pyc_files:
full_path = os.path.join(root, excess_pyc_file)
log.debug('Removing old PYC file: %s', full_path)
try:
os.remove(full_path)
except:
log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
if len(os.listdir(full_path)) == 0:
try:
os.rmdir(full_path)
except:
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
def cleanup(self):
# Wait a bit after starting before cleanup
log.debug('Cleaning up unused files')
try:
db = get_db()
cache_dir = Env.get('cache_dir')
medias = db.all('media', with_doc = True)
files = []
for media in medias:
file_dict = media['doc'].get('files', {})
for x in file_dict.keys():
files.extend(file_dict[x])
for f in os.listdir(cache_dir):
if os.path.splitext(f)[1] in ['.png', '.jpg', '.jpeg']:
file_path = os.path.join(cache_dir, f)
if toUnicode(file_path) not in files:
os.remove(file_path)
except:
log.error('Failed removing unused file: %s', traceback.format_exc())
def fromOld(self):
if request.method != 'POST':
return self.renderTemplate(__file__, 'form.html', url_for=url_for)
file = request.files['old_db']
uploaded_file = os.path.join(Env.get('cache_dir'), 'v1_database.db')
if os.path.isfile(uploaded_file):
os.remove(uploaded_file)
file.save(uploaded_file)
try:
import sqlite3
conn = sqlite3.connect(uploaded_file)
wanted = []
t = ('want', )
cur = conn.execute('SELECT status, imdb FROM Movie WHERE status=?',
t)
for row in cur:
status, imdb = row
if getImdb(imdb):
wanted.append(imdb)
conn.close()
wanted = set(wanted)
for imdb in wanted:
fireEventAsync('movie.add', {'identifier': imdb},
search_after=False)
message = 'Successfully imported %s movie(s)' % len(wanted)
except Exception, e:
message = 'Failed: %s' % e
def partial(self, type='all', lines=30, **kwargs):
total_lines = tryInt(lines)
log_lines = []
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
# Check see if the log exists
if not os.path.isfile(path):
break
f = open(path, 'r')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
brk = False
for line in reversed_lines:
if type == 'all' or '%s ' % type.upper() in line:
log_lines.append(line)
if len(log_lines) >= total_lines:
brk = True
break
if brk:
break
log_lines.reverse()
return {
'success': True,
'log': '[0m\n'.join(log_lines),
}
def register(self):
if self.registered: return
try:
hostname = self.conf('hostname')
password = self.conf('password')
port = self.conf('port')
self.growl = notifier.GrowlNotifier(
applicationName=Env.get('appname'),
notifications=['Updates'],
defaultNotifications=['Updates'],
applicationIcon=self.getNotificationImage('medium'),
hostname=hostname if hostname else 'localhost',
password=password if password else None,
port=port if port else 23053)
self.growl.register()
self.registered = True
except Exception as e:
if 'timed out' in str(e):
self.registered = True
else:
log.error('Failed register of growl: %s',
traceback.format_exc())
def cleanup(self):
# Wait a bit after starting before cleanup
time.sleep(3)
log.debug('Cleaning up unused files')
try:
db = get_session()
for root, dirs, walk_files in os.walk(Env.get('cache_dir')):
for filename in walk_files:
if os.path.splitext(filename)[1] in [
'.png', '.jpg', '.jpeg'
]:
file_path = os.path.join(root, filename)
f = db.query(File).filter(
File.path == toUnicode(file_path)).first()
if not f:
os.remove(file_path)
except:
log.error('Failed removing unused file: %s',
traceback.format_exc())
db.rollback()
finally:
db.close()
def safeMessage(self, msg, replace_tuple=()):
from couchpotato.environment import Env
from couchpotato.core.helpers.encoding import ss
msg = ss(msg)
try:
msg = msg % replace_tuple
except:
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) for x in list(replace_tuple)])
else:
msg = msg % ss(replace_tuple)
except:
self.logger.error(u'Failed encoding stuff to log: %s' %
traceback.format_exc())
if not Env.get('dev'):
for replace in self.replace_private:
msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace,
msg)
msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace,
msg)
# Replace api key
try:
api_key = Env.setting('api_key')
if api_key:
msg = msg.replace(api_key, 'API_KEY')
except:
pass
return msg
def urlopen(self,
url,
timeout=30,
data=None,
headers=None,
files=None,
show_error=True,
stream=False):
url = quote(ss(url), safe="%/:=&?~#+!$,;'@()*[]")
if not headers: headers = {}
if not data: data = {}
# Fill in some headers
parsed_url = urlparse(url)
host = '%s%s' % (parsed_url.hostname,
(':' +
str(parsed_url.port) if parsed_url.port else ''))
headers['Referer'] = headers.get('Referer',
'%s://%s' % (parsed_url.scheme, host))
headers['Host'] = headers.get('Host', None)
headers['User-Agent'] = headers.get('User-Agent', self.user_agent)
headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip')
headers['Connection'] = headers.get('Connection', 'keep-alive')
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
if headers.get('Authorization', '') != '':
headers['Authorization'] = headers.get('Authorization', '')
use_proxy = Env.setting('use_proxy')
proxy_url = None
if use_proxy:
proxy_server = Env.setting('proxy_server')
proxy_username = Env.setting('proxy_username')
proxy_password = Env.setting('proxy_password')
if proxy_server:
loc = "{0}:{1}@{2}".format(
proxy_username, proxy_password,
proxy_server) if proxy_username else proxy_server
proxy_url = {
"http": "http://" + loc,
"https": "https://" + loc,
}
else:
proxy_url = getproxies()
r = Env.get('http_opener')
# Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900):
log.info2(
'Disabled calls to %s for 15 minutes because so many failed requests.',
host)
if not show_error:
raise Exception(
'Disabled calls to %s for 15 minutes because so many failed requests'
% host)
else:
return ''
else:
del self.http_failed_request[host]
del self.http_failed_disabled[host]
self.wait(host, url)
status_code = None
try:
kwargs = {
'headers': headers,
'data': data if len(data) > 0 else None,
'timeout': timeout,
'files': files,
'verify':
False, #verify_ssl, Disable for now as to many wrongly implemented certificates..
'stream': stream,
'proxies': proxy_url,
}
method = 'post' if len(data) > 0 or files else 'get'
log.info('Opening url: %s %s, data: %s',
(method, url, [x for x in data.keys()] if isinstance(
data, dict) else 'with data'))
response = r.request(method, url, **kwargs)
status_code = response.status_code
if response.status_code == requests.codes.ok:
data = response if stream else response.content
else:
response.raise_for_status()
self.http_failed_request[host] = 0
except (IOError, MaxRetryError, Timeout):
if show_error:
log.error('Failed opening url in %s: %s %s',
(self.getName(), url, traceback.format_exc(0)))
# Save failed requests by hosts
try:
# To many requests
if status_code in [429]:
self.http_failed_request[host] = 1
self.http_failed_disabled[host] = time.time()
if not self.http_failed_request.get(host):
self.http_failed_request[host] = 1
else:
self.http_failed_request[host] += 1
# Disable temporarily
if self.http_failed_request[host] > 5 and not isLocalIP(
host):
self.http_failed_disabled[host] = time.time()
except:
log.debug('Failed logging failed requests for %s: %s',
(url, traceback.format_exc()))
raise
self.http_last_use[host] = time.time()
return data
def __init__(self):
self.desktop = Env.get('desktop')
def __init__(self):
# Create version file in cache
self.version_file = os.path.join(Env.get('cache_dir'), 'version')
if not os.path.isfile(self.version_file):
self.createFile(self.version_file, json.dumps(self.latestCommit()))
def setCache(self, cache_key, value, timeout=300):
cache_key_md5 = md5(cache_key)
log.debug('Setting cache %s', cache_key)
Env.get('cache').set(cache_key_md5, value, timeout)
return value
def get_db():
return Env.get('db')
def get(self, *args, **kwargs):
self.clear_cookie('user')
self.redirect('%slogin/' % Env.get('web_base'))
def get(self, *args, **kwargs):
if self.get_current_user():
self.redirect(Env.get('web_base'))
else:
self.write(template_loader.load('login.html').generate(sep = os.sep, fireEvent = fireEvent, Env = Env))
def ss(original, *args):
from couchpotato.environment import Env
return toUnicode(original, *args).encode(Env.get('encoding'))
def __init__(self, git_command):
self.repo = LocalRepository(Env.get('app_dir'), command = git_command)
