def getCache(self, cache_key, url=None, **kwargs): use_cache = not len(kwargs.get("data", {})) > 0 and not kwargs.get("files") if use_cache: cache_key_md5 = md5(cache_key) cache = Env.get("cache").get(cache_key_md5) if cache: if not Env.get("dev"): log.debug("Getting cache %s", cache_key) return cache if url: try: cache_timeout = 300 if "cache_timeout" in kwargs: cache_timeout = kwargs.get("cache_timeout") del kwargs["cache_timeout"] data = self.urlopen(url, **kwargs) if data and cache_timeout > 0 and use_cache: self.setCache(cache_key, data, timeout=cache_timeout) return data except: if not kwargs.get("show_error", True): raise log.debug("Failed getting cache: %s", (traceback.format_exc(0))) return ""
def replaceWith(self, path): app_dir = Env.get('app_dir') # Get list of files we want to overwrite self.deletePyc(only_excess = False) existing_files = [] for root, subfiles, filenames in os.walk(app_dir): for filename in filenames: existing_files.append(os.path.join(root, filename)) for root, subfiles, filenames in os.walk(path): for filename in filenames: fromfile = os.path.join(root, filename) tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, '')) if not Env.get('dev'): try: os.remove(tofile) except: pass try: os.renames(fromfile, tofile) try: existing_files.remove(tofile) except ValueError: pass except Exception, e: log.error('Failed overwriting file: %s' % e)
def __init__(self): if Env.get('desktop'): self.updater = DesktopUpdater() elif os.path.isdir(os.path.join(Env.get('app_dir'), '.git')): self.updater = GitUpdater(self.conf('git_command', default = 'git')) else: self.updater = SourceUpdater() fireEvent('schedule.interval', 'updater.check', self.autoUpdate, hours = 6) addEvent('app.load', self.autoUpdate) addEvent('updater.info', self.info) addApiView('updater.info', self.getInfo, docs = { 'desc': 'Get updater information', 'return': { 'type': 'object', 'example': """{ 'last_check': "last checked for update", 'update_version': "available update version or empty", 'version': current_cp_version }"""} }) addApiView('updater.update', self.doUpdateView) addApiView('updater.check', self.checkView, docs = { 'desc': 'Check for available update', 'return': {'type': 'see updater.info'} })
def getCache(self, cache_key, url = None, **kwargs): cache_key = simplifyString(cache_key) cache = Env.get('cache').get(cache_key) if cache: if not Env.get('dev'): log.debug('Getting cache %s', cache_key) return cache if url: try: cache_timeout = 300 if kwargs.get('cache_timeout'): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] opener = None if kwargs.get('opener'): opener = kwargs.get('opener') del kwargs['opener'] if opener: log.info('Opening url: %s', url) f = opener.open(url) data = f.read() f.close() else: data = self.urlopen(url, **kwargs) if data: self.setCache(cache_key, data, timeout = cache_timeout) return data except: pass
def registerStatic(self, plugin_file, add_to_head=True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) static_folder = toUnicode(os.path.join(self.plugin_path, "static")) if not os.path.isdir(static_folder): return # Get plugin_name from PluginName s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__) class_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() # View path path = "static/plugin/%s/" % class_name # Add handler to Tornado Env.get("app").add_handlers( ".*$", [(Env.get("web_base") + path + "(.*)", StaticFileHandler, {"path": static_folder})] ) # Register for HTML <HEAD> if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, "static", "*")): ext = getExt(f) if ext in ["js", "css"]: fireEvent("register_%s" % ("script" if ext in "js" else "style"), path + os.path.basename(f), f)
def doUpdate(self): try: url = 'https://github.com/%s/%s/tarball/%s' % (self.repo_user, self.repo_name, self.branch) destination = os.path.join(Env.get('cache_dir'), self.update_version.get('hash') + '.tar.gz') extracted_path = os.path.join(Env.get('cache_dir'), 'temp_updater') destination = fireEvent('file.download', url = url, dest = destination, single = True) # Cleanup leftover from last time if os.path.isdir(extracted_path): self.removeDir(extracted_path) self.makeDir(extracted_path) # Extract tar = tarfile.open(destination) tar.extractall(path = extracted_path) tar.close() os.remove(destination) if self.replaceWith(os.path.join(extracted_path, os.listdir(extracted_path)[0])): self.removeDir(extracted_path) # Write update version to file self.createFile(self.version_file, json.dumps(self.update_version)) return True except: log.error('Failed updating: %s', traceback.format_exc()) self.update_failed = True return False
def registerStatic(self, plugin_file, add_to_head = True): # Register plugin path self.plugin_path = os.path.dirname(plugin_file) static_folder = toUnicode(os.path.join(self.plugin_path, 'static')) if not os.path.isdir(static_folder): return # Get plugin_name from PluginName s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__) class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() # View path path = 'static/plugin/%s/' % class_name # Add handler to Tornado Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})]) # Register for HTML <HEAD> if add_to_head: for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')): ext = getExt(f) if ext in ['js', 'css']: fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f)
def __init__(self): if Env.get("desktop"): self.updater = DesktopUpdater() elif os.path.isdir(os.path.join(Env.get("app_dir"), ".git")): self.updater = GitUpdater(self.conf("git_command", default="git")) else: self.updater = SourceUpdater() addEvent("app.load", self.setCrons) addEvent("updater.info", self.info) addApiView( "updater.info", self.getInfo, docs={ "desc": "Get updater information", "return": { "type": "object", "example": """{ 'last_check': "last checked for update", 'update_version': "available update version or empty", 'version': current_cp_version }""", }, }, ) addApiView("updater.update", self.doUpdateView) addApiView( "updater.check", self.checkView, docs={"desc": "Check for available update", "return": {"type": "see updater.info"}}, ) addEvent("setting.save.updater.enabled.after", self.setCrons)
def getCache(self, cache_key, url=None, **kwargs): cache_key_md5 = md5(cache_key) cache = Env.get("cache").get(cache_key_md5) if cache: if not Env.get("dev"): log.debug("Getting cache %s", cache_key) return cache if url: try: cache_timeout = 300 if kwargs.get("cache_timeout"): cache_timeout = kwargs.get("cache_timeout") del kwargs["cache_timeout"] data = self.urlopen(url, **kwargs) if data: self.setCache(cache_key, data, timeout=cache_timeout) return data except: if not kwargs.get("show_error", True): raise return ""
def getCache(self, cache_key, url = None, **kwargs): use_cache = not len(kwargs.get('data', {})) > 0 and not kwargs.get('files') if use_cache: cache_key_md5 = md5(cache_key) cache = Env.get('cache').get(cache_key_md5) if cache: if not Env.get('dev'): log.debug('Getting cache %s', cache_key) return cache if url: try: cache_timeout = 300 if 'cache_timeout' in kwargs: cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] data = self.urlopen(url, **kwargs) if data and cache_timeout > 0 and use_cache: self.setCache(cache_key, data, timeout = cache_timeout) return data except: if not kwargs.get('show_error', True): raise log.debug('Failed getting cache: %s', (traceback.format_exc(0))) return ''
def getCache(self, cache_key, url = None, **kwargs): cache_key = md5(ss(cache_key)) cache = Env.get('cache').get(cache_key) if cache: if not Env.get('dev'): log.debug('Getting cache %s', cache_key) return cache if url: try: cache_timeout = 300 if kwargs.get('cache_timeout'): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] data = self.urlopen(url, **kwargs) if data: self.setCache(cache_key, data, timeout = cache_timeout) return data except: if not kwargs.get('show_error', True): raise return ''
def getUserScript(self, script_route, **kwargs): klass = self class UserscriptHandler(RequestHandler): def get(self, random, route): bookmarklet_host = Env.setting('bookmarklet_host') loc = bookmarklet_host if bookmarklet_host else "{0}://{1}".format(self.request.protocol, self.request.headers.get('X-Forwarded-Host') or self.request.headers.get('host')) params = { 'includes': fireEvent('userscript.get_includes', merge = True), 'excludes': fireEvent('userscript.get_excludes', merge = True), 'version': klass.getVersion(), 'api': '%suserscript/' % Env.get('api_base'), 'host': loc, } script = klass.renderTemplate(__file__, 'template.js_tmpl', **params) klass.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script) self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js') Env.get('app').add_handlers(".*$", [('%s%s' % (Env.get('api_base'), script_route), UserscriptHandler)])
def __init__(self): if Env.get('desktop'): self.updater = DesktopUpdater() elif os.path.isdir(os.path.join(Env.get('app_dir'), '.git')): self.updater = GitUpdater(self.conf('git_command', default = 'git')) else: self.updater = SourceUpdater() addEvent('app.load', self.logVersion, priority = 10000) addEvent('app.load', self.setCrons) addEvent('updater.info', self.info) addApiView('updater.info', self.info, docs = { 'desc': 'Get updater information', 'return': { 'type': 'object', 'example': """{ 'last_check': "last checked for update", 'update_version': "available update version or empty", 'version': current_cp_version }"""} }) addApiView('updater.update', self.doUpdateView) addApiView('updater.check', self.checkView, docs = { 'desc': 'Check for available update', 'return': {'type': 'see updater.info'} }) addEvent('setting.save.updater.enabled.after', self.setCrons)
def checkDataDir(self): if Env.get("app_dir") in Env.get("data_dir"): log.error( "You should NOT use your CouchPotato directory to save your settings in. Files will get overwritten or be deleted." ) return True
def getCredentials(self, **kwargs): try: oauth_token = kwargs.get('oauth') except: return 'redirect', Env.get('web_base') + 'settings/downloaders/' log.debug('oauth_token is: %s', oauth_token) self.conf('oauth_token', value = oauth_token); return 'redirect', Env.get('web_base') + 'settings/downloaders/'
def search(self): params = getParams() cache_key = "%s/%s" % (__name__, urlencode(params)) movies = Env.get("cache").get(cache_key) if not movies: movies = fireEvent("movie.search", q=params.get("q"), merge=True) Env.get("cache").set(cache_key, movies) return jsonified({"success": True, "empty": len(movies) == 0 if movies else 0, "movies": movies})
def ss(original, *args): u_original = toUnicode(original, *args) try: from couchpotato.environment import Env return u_original.encode(Env.get('encoding')) except Exception as e: log.debug('Failed ss encoding char, force UTF8: %s', e) try: return u_original.encode(Env.get('encoding'), 'replace') except: return u_original.encode('utf-8', 'replace')
def getCache(self, cache_key, url = None, timeout = 300): cache = Env.get('cache').get(cache_key) if cache: if not Env.get('dev'): log.debug('Getting cache %s' % cache_key) return cache if url: try: data = self.urlopen(url) self.setCache(cache_key, data, timeout = timeout) return data except: pass
def page_not_found(rh): index_url = Env.get('web_base') url = rh.request.uri[len(index_url):] if url[:3] != 'api': r = index_url + '#' + url.lstrip('/') rh.redirect(r) else: if not Env.get('dev'): time.sleep(0.1) rh.set_status(404) rh.write('Wrong API key used')
def get(self, random, route): params = { 'includes': fireEvent('userscript.get_includes', merge = True), 'excludes': fireEvent('userscript.get_excludes', merge = True), 'version': klass.getVersion(), 'api': '%suserscript/' % Env.get('api_base'), 'host': '%s://%s' % (self.request.protocol, self.request.host), } script = klass.renderTemplate(__file__, 'template.js', **params) klass.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script) self.redirect(Env.get('api_base') + 'file.cache/couchpotato.user.js')
def ss(original, *args): u_original = toUnicode(original, *args) try: from couchpotato.environment import Env return u_original.encode(Env.get("encoding")) except Exception as e: log.debug("Failed ss encoding char, force UTF8: %s", e) return u_original.encode("utf8") try: return u_original.encode(Env.get("encoding"), "replace") except: return u_original.encode("utf-8", "replace")
def minify(self): # Create cache dir cache = Env.get('cache_dir') parent_dir = os.path.join(cache, 'minified') self.makeDir(parent_dir) Env.get('app').add_handlers(".*$", [(Env.get('web_base') + 'minified/(.*)', StaticFileHandler, {'path': parent_dir})]) for file_type in ['style', 'script']: ext = 'js' if file_type is 'script' else 'css' positions = self.paths.get(file_type, {}) for position in positions: files = positions.get(position) self._minify(file_type, files, position, position + '.' + ext)
def search(self): q = getParam("q") cache_key = u"%s/%s" % (__name__, simplifyString(q)) movies = Env.get("cache").get(cache_key) if not movies: if getImdb(q): movies = [fireEvent("movie.info", identifier=q, merge=True)] else: movies = fireEvent("movie.search", q=q, merge=True) Env.get("cache").set(cache_key, movies) return jsonified({"success": True, "empty": len(movies) == 0 if movies else 0, "movies": movies})
def search(self): params = getParams() cache_key = '%s/%s' % (__name__, urlencode(params)) movies = Env.get('cache').get(cache_key) if not movies: movies = fireEvent('movie.search', q = params.get('q'), merge = True) Env.get('cache').set(cache_key, movies) return jsonified({ 'success': True, 'empty': len(movies) == 0 if movies else 0, 'movies': movies, })
def getUserScript(self, filename = ''): params = { 'includes': fireEvent('userscript.get_includes', merge = True), 'excludes': fireEvent('userscript.get_excludes', merge = True), 'version': self.getVersion(), 'api': '%suserscript/' % url_for('api.index').lstrip('/'), 'host': request.host_url, } script = self.renderTemplate(__file__, 'template.js', **params) self.createFile(os.path.join(Env.get('cache_dir'), 'couchpotato.user.js'), script) from flask.helpers import send_from_directory return send_from_directory(Env.get('cache_dir'), 'couchpotato.user.js')
def fromOld(self): if request.method != 'POST': return self.renderTemplate(__file__, 'form.html', url_for = url_for) file = request.files['old_db'] uploaded_file = os.path.join(Env.get('cache_dir'), 'v1_database.db') if os.path.isfile(uploaded_file): os.remove(uploaded_file) file.save(uploaded_file) try: import sqlite3 conn = sqlite3.connect(uploaded_file) wanted = [] t = ('want',) cur = conn.execute('SELECT status, imdb FROM Movie WHERE status=?', t) for row in cur: status, imdb = row if getImdb(imdb): wanted.append(imdb) conn.close() wanted = set(wanted) for imdb in wanted: fireEventAsync('movie.add', {'identifier': imdb}, search_after = False) message = 'Successfully imported %s movie(s)' % len(wanted) except Exception, e: message = 'Failed: %s' % e
def check(self): if self.update_version: return True log.info('Checking for new version on github for %s', self.repo_name) if not Env.get('dev'): self.repo.fetch() current_branch = self.repo.getCurrentBranch().name for branch in self.repo.getRemoteByName('origin').getBranches(): if current_branch == branch.name: local = self.repo.getHead() remote = branch.getHead() log.info('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8])) if local.getDate() < remote.getDate(): self.update_version = { 'hash': remote.hash[:8], 'date': remote.getDate(), } return True self.last_check = time.time() return False
def showCacheFile(self, filename = ''): cache_dir = Env.get('cache_dir') filename = os.path.basename(filename) from flask.helpers import send_from_directory return send_from_directory(cache_dir, filename)
def __init__(self): fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2) addEvent('manage.update', self.updateLibrary) addEvent('manage.diskspace', self.getDiskSpace) # Add files after renaming def after_rename(message = None, group = None): if not group: group = {} return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files']) addEvent('renamer.after', after_rename, priority = 110) addApiView('manage.update', self.updateLibraryView, docs = { 'desc': 'Update the library by scanning for new movies', 'params': { 'full': {'desc': 'Do a full update or just recently changed/added movies.'}, } }) addApiView('manage.progress', self.getProgress, docs = { 'desc': 'Get the progress of current manage update', 'return': {'type': 'object', 'example': """{ 'progress': False || object, total & to_go, }"""}, }) if not Env.get('dev') and self.conf('startup_scan'): addEvent('app.load', self.updateLibraryQuick)
def safeMessage(self, msg, replace_tuple = ()): from couchpotato.environment import Env from couchpotato.core.helpers.encoding import ss msg = ss(msg) try: msg = msg % replace_tuple except: try: if isinstance(replace_tuple, tuple): msg = msg % tuple([ss(x) for x in list(replace_tuple)]) else: msg = msg % ss(replace_tuple) except: self.logger.error(u'Failed encoding stuff to log: %s' % traceback.format_exc()) if not Env.get('dev'): for replace in self.replace_private: msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace, msg) msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace, msg) # Replace api key try: api_key = Env.setting('api_key') if api_key: msg = msg.replace(api_key, 'API_KEY') except: pass return msg
def download(self, url='', dest=None, overwrite=False): try: file = urllib2.urlopen(url) if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if overwrite or not os.path.exists(dest): log.debug('Writing file to: %s' % dest) output = open(dest, 'wb') output.write(file.read()) output.close() else: log.debug('File already exists: %s' % dest) return dest except Exception, e: log.error('Unable to download file "%s": %s' % (url, e))
def download(self, data=None, media=None, filedata=None): if not media: media = {} if not data: data = {} log.info('Sending "%s" to put.io', data.get('name')) url = data.get('url') client = pio.Client(self.conf('oauth_token')) putioFolder = self.convertFolder(client, self.conf('folder')) log.debug('putioFolder ID is %s', putioFolder) # It might be possible to call getFromPutio from the renamer if we can then we don't need to do this. # Note callback_host is NOT our address, it's the internet host that putio can call too callbackurl = None if self.conf('download'): callbackurl = 'http://' + self.conf( 'callback_host') + '%sdownloader.putio.getfrom/' % Env.get( 'api_base'.strip('/')) resp = client.Transfer.add_url(url, callback_url=callbackurl, parent_id=putioFolder) log.debug('resp is %s', resp.id) return self.downloadReturnId(resp.id)
def clear(self, **kwargs): for x in range(0, 50): path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '') if not os.path.isfile(path): continue try: # Create empty file for current logging if x is 0: self.createFile(path, '') else: os.remove(path) except: log.error('Couldn\'t delete file "%s": %s', (path, traceback.format_exc())) return {'success': True}
def searchLibrary(self): # Get all active and online movies db = get_session() library = db.query(Library).all() done_status = fireEvent('status.get', 'done', single = True) for movie in library.movies: for release in movie.releases: # get releases and their movie files if release.status_id is done_status.get('id'): files = [] for file in release.files.filter(FileType.status.has(identifier = 'movie')).all(): files.append(file.path) # get subtitles for those files subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services)
def register(self): if self.registered: return try: hostname = self.conf('hostname') password = self.conf('password') port = self.conf('port') self.growl = notifier.GrowlNotifier( applicationName=Env.get('appname'), notifications=["Updates"], defaultNotifications=["Updates"], applicationIcon='%s/static/images/couch.png' % fireEvent('app.api_url', single=True), hostname=hostname if hostname else 'localhost', password=password if password else None, port=port if port else 23053) self.growl.register() self.registered = True except: log.error('Failed register of growl: %s', traceback.format_exc())
def partial(self, type = 'all', lines = 30, offset = 0, **kwargs): total_lines = try_int(lines) offset = try_int(offset) log_lines = [] for x in range(0, 50): path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '') # Check see if the log exists if not os.path.isfile(path): break f = open(path, 'r') log_content = to_unicode(f.read()) raw_lines = self.toList(log_content) raw_lines.reverse() brk = False for line in raw_lines: if type == 'all' or line.get('type') == type.upper(): log_lines.append(line) if len(log_lines) >= (total_lines + offset): brk = True break if brk: break log_lines = log_lines[offset:] log_lines.reverse() return { 'success': True, 'log': log_lines, }
def __init__(self): super(CoreNotifier, self).__init__() addEvent('notify', self.notify) addEvent('notify.frontend', self.frontend) addApiView('notification.markread', self.markAsRead, docs = { 'desc': 'Mark notifications as read', 'params': { 'ids': {'desc': 'Notification id you want to mark as read. All if ids is empty.', 'type': 'int (comma separated)'}, }, }) addApiView('notification.list', self.listView, docs = { 'desc': 'Get list of notifications', 'params': { 'limit_offset': {'desc': 'Limit and offset the notification list. Examples: "50" or "50,30"'}, }, 'return': {'type': 'object', 'example': """{ 'success': True, 'empty': bool, any notification returned or not, 'notifications': array, notifications found, }"""} }) addNonBlockApiView('notification.listener', (self.addListener, self.removeListener)) addApiView('notification.listener', self.listener) fireEvent('schedule.interval', 'core.check_messages', self.checkMessages, hours = 12, single = True) fireEvent('schedule.interval', 'core.clean_messages', self.cleanMessages, seconds = 15, single = True) addEvent('app.load', self.clean) if not Env.get('dev'): addEvent('app.load', self.checkMessages) self.messages = [] self.listeners = [] self.m_lock = threading.Lock()
def saveView(self, **kwargs): section = kwargs.get('section') option = kwargs.get('name') value = kwargs.get('value') if not self.isOptionWritable(section, option): self.log.warning('Option "%s.%s" isn\'t writable', (section, option)) return { 'success' : False, } from couchpotato.environment import Env soft_chroot = Env.get('softchroot') if self.getType(section, option) == 'directory': value = soft_chroot.chroot2abs(value) if self.getType(section, option) == 'directories': import json value = json.loads(value) if not (value and isinstance(value, list)): value = [] value = map(soft_chroot.chroot2abs, value) value = self.directories_delimiter.join(value) # See if a value handler is attached, use that as value new_value = fireEvent('setting.save.%s.%s' % (section, option), value, single = True) self.set(section, option, (new_value if new_value else value).encode('unicode_escape')) self.save() # After save (for re-interval etc) fireEvent('setting.save.%s.%s.after' % (section, option), single = True) fireEvent('setting.save.%s.*.after' % section, single = True) return { 'success': True }
def _minify(self, file_type, files, position, out): cache = Env.get('cache_dir') out_name = 'minified_' + out out = os.path.join(cache, out_name) raw = [] for file_path in files: f = open(file_path, 'r').read() if file_type == 'script': data = jsmin(f) else: data = cssmin(f) data = data.replace('../images/', '../static/images/') raw.append({ 'file': file_path, 'date': int(os.path.getmtime(file_path)), 'data': data }) # Combine all files together with some comments data = '' for r in raw: data += self.comment.get(file_type) % (r.get('file'), r.get('date')) data += r.get('data') + '\n\n' self.createFile(out, data.strip()) if not self.minified.get(file_type): self.minified[file_type] = {} if not self.minified[file_type].get(position): self.minified[file_type][position] = [] minified_url = 'api/%s/file.cache/%s?%s' % ( Env.setting('api_key'), out_name, tryInt(os.path.getmtime(out))) self.minified[file_type][position].append(minified_url)
def searchSingle(self, group): if self.isDisabled(): return try: available_languages = sum(group['subtitle_language'].itervalues(), []) downloaded = [] files = [toUnicode(x) for x in group['files']['movie']] log.debug('Searching for subtitles for: %s', files) for lang in self.getLanguages(): if lang not in available_languages: download = subliminal.download_subtitles( files, multi=True, force=False, languages=[lang], services=self.services, cache_dir=Env.get('cache_dir')) for subtitle in download: downloaded.extend(download[subtitle]) for d_sub in downloaded: log.info('Found subtitle (%s): %s', (d_sub.language.alpha2, files)) group['files']['subtitle'].append(d_sub.path) group['subtitle_language'][d_sub.path] = [ d_sub.language.alpha2 ] return True except: log.error('Failed searching for subtitle: %s', (traceback.format_exc())) return False
def partial(self): log_type = getParam('type', 'all') total_lines = tryInt(getParam('lines', 30)) log_lines = [] for x in range(0, 50): path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '') # Check see if the log exists if not os.path.isfile(path): break reversed_lines = [] f = open(path, 'r') reversed_lines = toUnicode(f.read()).split('[0m\n') reversed_lines.reverse() brk = False for line in reversed_lines: if log_type == 'all' or '%s ' % log_type.upper() in line: log_lines.append(line) if len(log_lines) >= total_lines: brk = True break if brk: break log_lines.reverse() return jsonified({ 'success': True, 'log': '[0m\n'.join(log_lines), })
def __init__(self): # Get options via arg from couchpotato.runner import getOptions self.options = getOptions(base_path, sys.argv[1:]) # Load settings settings = Env.get('settings') settings.setFile(self.options.config_file) # Create data dir if needed if self.options.data_dir: self.data_dir = self.options.data_dir else: self.data_dir = os.path.expanduser(Env.setting('data_dir')) if self.data_dir == '': self.data_dir = getDataDir() if not os.path.isdir(self.data_dir): os.makedirs(self.data_dir) # Create logging dir self.log_dir = os.path.join(self.data_dir, 'logs') if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir) # Logging from couchpotato.core.logger import CPLog self.log = CPLog(__name__) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S') hdlr = handlers.RotatingFileHandler( os.path.join(self.log_dir, 'error.log'), 'a', 500000, 10) hdlr.setLevel(logging.CRITICAL) hdlr.setFormatter(formatter) self.log.logger.addHandler(hdlr)
def __init__(self): addApiView('app.shutdown', self.shutdown, docs={ 'desc': 'Shutdown the app.', 'return': { 'type': 'string: shutdown' } }) addApiView('app.restart', self.restart, docs={ 'desc': 'Restart the app.', 'return': { 'type': 'string: restart' } }) addApiView('app.available', self.available, docs={'desc': 'Check if app available.'}) addApiView('app.version', self.versionView, docs={'desc': 'Get version.'}) addEvent('app.shutdown', self.shutdown) addEvent('app.restart', self.restart) addEvent('app.load', self.launchBrowser, priority=1) addEvent('app.base_url', self.createBaseUrl) addEvent('app.api_url', self.createApiUrl) addEvent('app.version', self.version) addEvent('app.load', self.checkDataDir) addEvent('setting.save.core.password', self.md5Password) addEvent('setting.save.core.api_key', self.checkApikey) # Make sure we can close-down with ctrl+c properly if not Env.get('desktop'): self.signalHandler()
def download(self, url='', dest=None, overwrite=False, urlopen_kwargs=None): if not urlopen_kwargs: urlopen_kwargs = {} if not dest: # to Cache dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url))) if not overwrite and os.path.isfile(dest): return dest try: filedata = self.urlopen(url, **urlopen_kwargs) except: log.error('Failed downloading file %s: %s', (url, traceback.format_exc())) return False self.createFile(dest, filedata, binary=True) return dest
def deletePyc(self, only_excess = True): for root, dirs, files in os.walk(ss(Env.get('app_dir'))): pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) py_files = set(filter(lambda filename: filename.endswith('.py'), files)) excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files for excess_pyc_file in excess_pyc_files: full_path = os.path.join(root, excess_pyc_file) log.debug('Removing old PYC file: %s', full_path) try: os.remove(full_path) except: log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc())) for dir_name in dirs: full_path = os.path.join(root, dir_name) if len(os.listdir(full_path)) == 0: try: os.rmdir(full_path) except: log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
def cleanup(self): # Wait a bit after starting before cleanup log.debug('Cleaning up unused files') try: db = get_db() cache_dir = Env.get('cache_dir') medias = db.all('media', with_doc = True) files = [] for media in medias: file_dict = media['doc'].get('files', {}) for x in file_dict.keys(): files.extend(file_dict[x]) for f in os.listdir(cache_dir): if os.path.splitext(f)[1] in ['.png', '.jpg', '.jpeg']: file_path = os.path.join(cache_dir, f) if toUnicode(file_path) not in files: os.remove(file_path) except: log.error('Failed removing unused file: %s', traceback.format_exc())
def fromOld(self): if request.method != 'POST': return self.renderTemplate(__file__, 'form.html', url_for=url_for) file = request.files['old_db'] uploaded_file = os.path.join(Env.get('cache_dir'), 'v1_database.db') if os.path.isfile(uploaded_file): os.remove(uploaded_file) file.save(uploaded_file) try: import sqlite3 conn = sqlite3.connect(uploaded_file) wanted = [] t = ('want', ) cur = conn.execute('SELECT status, imdb FROM Movie WHERE status=?', t) for row in cur: status, imdb = row if getImdb(imdb): wanted.append(imdb) conn.close() wanted = set(wanted) for imdb in wanted: fireEventAsync('movie.add', {'identifier': imdb}, search_after=False) message = 'Successfully imported %s movie(s)' % len(wanted) except Exception, e: message = 'Failed: %s' % e
def partial(self, type='all', lines=30, **kwargs): total_lines = tryInt(lines) log_lines = [] for x in range(0, 50): path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '') # Check see if the log exists if not os.path.isfile(path): break f = open(path, 'r') reversed_lines = toUnicode(f.read()).split('[0m\n') reversed_lines.reverse() brk = False for line in reversed_lines: if type == 'all' or '%s ' % type.upper() in line: log_lines.append(line) if len(log_lines) >= total_lines: brk = True break if brk: break log_lines.reverse() return { 'success': True, 'log': '[0m\n'.join(log_lines), }
def register(self): if self.registered: return try: hostname = self.conf('hostname') password = self.conf('password') port = self.conf('port') self.growl = notifier.GrowlNotifier( applicationName=Env.get('appname'), notifications=['Updates'], defaultNotifications=['Updates'], applicationIcon=self.getNotificationImage('medium'), hostname=hostname if hostname else 'localhost', password=password if password else None, port=port if port else 23053) self.growl.register() self.registered = True except Exception as e: if 'timed out' in str(e): self.registered = True else: log.error('Failed register of growl: %s', traceback.format_exc())
def cleanup(self): # Wait a bit after starting before cleanup time.sleep(3) log.debug('Cleaning up unused files') try: db = get_session() for root, dirs, walk_files in os.walk(Env.get('cache_dir')): for filename in walk_files: if os.path.splitext(filename)[1] in [ '.png', '.jpg', '.jpeg' ]: file_path = os.path.join(root, filename) f = db.query(File).filter( File.path == toUnicode(file_path)).first() if not f: os.remove(file_path) except: log.error('Failed removing unused file: %s', traceback.format_exc()) db.rollback() finally: db.close()
def safeMessage(self, msg, replace_tuple=()): from couchpotato.environment import Env from couchpotato.core.helpers.encoding import ss msg = ss(msg) try: msg = msg % replace_tuple except: try: if isinstance(replace_tuple, tuple): msg = msg % tuple([ss(x) for x in list(replace_tuple)]) else: msg = msg % ss(replace_tuple) except: self.logger.error(u'Failed encoding stuff to log: %s' % traceback.format_exc()) if not Env.get('dev'): for replace in self.replace_private: msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace, msg) msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace, msg) # Replace api key try: api_key = Env.setting('api_key') if api_key: msg = msg.replace(api_key, 'API_KEY') except: pass return msg
def urlopen(self, url, timeout=30, data=None, headers=None, files=None, show_error=True, stream=False): url = quote(ss(url), safe="%/:=&?~#+!$,;'@()*[]") if not headers: headers = {} if not data: data = {} # Fill in some headers parsed_url = urlparse(url) host = '%s%s' % (parsed_url.hostname, (':' + str(parsed_url.port) if parsed_url.port else '')) headers['Referer'] = headers.get('Referer', '%s://%s' % (parsed_url.scheme, host)) headers['Host'] = headers.get('Host', None) headers['User-Agent'] = headers.get('User-Agent', self.user_agent) headers['Accept-encoding'] = headers.get('Accept-encoding', 'gzip') headers['Connection'] = headers.get('Connection', 'keep-alive') headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0') if headers.get('Authorization', '') != '': headers['Authorization'] = headers.get('Authorization', '') use_proxy = Env.setting('use_proxy') proxy_url = None if use_proxy: proxy_server = Env.setting('proxy_server') proxy_username = Env.setting('proxy_username') proxy_password = Env.setting('proxy_password') if proxy_server: loc = "{0}:{1}@{2}".format( proxy_username, proxy_password, proxy_server) if proxy_username else proxy_server proxy_url = { "http": "http://" + loc, "https": "https://" + loc, } else: proxy_url = getproxies() r = Env.get('http_opener') # Don't try for failed requests if self.http_failed_disabled.get(host, 0) > 0: if self.http_failed_disabled[host] > (time.time() - 900): log.info2( 'Disabled calls to %s for 15 minutes because so many failed requests.', host) if not show_error: raise Exception( 'Disabled calls to %s for 15 minutes because so many failed requests' % host) else: return '' else: del self.http_failed_request[host] del self.http_failed_disabled[host] self.wait(host, url) status_code = None try: kwargs = { 'headers': headers, 'data': data if len(data) > 0 else None, 'timeout': timeout, 'files': files, 'verify': False, #verify_ssl, Disable for now as to many wrongly implemented certificates.. 'stream': stream, 'proxies': proxy_url, } method = 'post' if len(data) > 0 or files else 'get' log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.keys()] if isinstance( data, dict) else 'with data')) response = r.request(method, url, **kwargs) status_code = response.status_code if response.status_code == requests.codes.ok: data = response if stream else response.content else: response.raise_for_status() self.http_failed_request[host] = 0 except (IOError, MaxRetryError, Timeout): if show_error: log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(0))) # Save failed requests by hosts try: # To many requests if status_code in [429]: self.http_failed_request[host] = 1 self.http_failed_disabled[host] = time.time() if not self.http_failed_request.get(host): self.http_failed_request[host] = 1 else: self.http_failed_request[host] += 1 # Disable temporarily if self.http_failed_request[host] > 5 and not isLocalIP( host): self.http_failed_disabled[host] = time.time() except: log.debug('Failed logging failed requests for %s: %s', (url, traceback.format_exc())) raise self.http_last_use[host] = time.time() return data
def __init__(self): self.desktop = Env.get('desktop')
def __init__(self): # Create version file in cache self.version_file = os.path.join(Env.get('cache_dir'), 'version') if not os.path.isfile(self.version_file): self.createFile(self.version_file, json.dumps(self.latestCommit()))
def setCache(self, cache_key, value, timeout=300): cache_key_md5 = md5(cache_key) log.debug('Setting cache %s', cache_key) Env.get('cache').set(cache_key_md5, value, timeout) return value
def get_db(): return Env.get('db')
def get(self, *args, **kwargs): self.clear_cookie('user') self.redirect('%slogin/' % Env.get('web_base'))
def get(self, *args, **kwargs): if self.get_current_user(): self.redirect(Env.get('web_base')) else: self.write(template_loader.load('login.html').generate(sep = os.sep, fireEvent = fireEvent, Env = Env))
def ss(original, *args): from couchpotato.environment import Env return toUnicode(original, *args).encode(Env.get('encoding'))
def __init__(self, git_command): self.repo = LocalRepository(Env.get('app_dir'), command = git_command)