def proxy_to_file(self, request, dest): """Handle local ``file:`` URLs""" orig_base = request.application_url ## FIXME: security restrictions here? assert dest.startswith('file:') if '?' in dest: dest = dest.split('?', 1)[0] filename = url_to_filename(dest) rest = posixpath.normpath(request.path_info) proxied_url = dest.lstrip('/') + '/' + urllib.parse.quote( rest.lstrip('/')) ## FIXME: handle /->/index.html filename = filename.rstrip('/') + '/' + rest.lstrip('/') if os.path.isdir(filename): if not request.path.endswith('/'): new_url = request.path + '/' if request.query_string: new_url += '?' + request.query_string resp = exc.HTTPMovedPermanently(location=new_url) return resp, orig_base, dest, proxied_url ## FIXME: configurable? StaticURLParser? for base in ['index.html', 'index.htm']: if os.path.exists(os.path.join(filename, base)): filename = os.path.join(filename, base) break else: resp = exc.HTTPNotFound( "There was no index.html file in the directory") if not os.path.exists(filename): resp = exc.HTTPNotFound("The file %s could not be found" % filename) else: app = FileApp(filename) resp = request.get_response(app) return resp, orig_base, dest, proxied_url
def proxy_to_file(self, request, dest): """Handle local ``file:`` URLs""" orig_base = request.application_url ## FIXME: security restrictions here? assert dest.startswith("file:") if "?" in dest: dest = dest.split("?", 1)[0] filename = url_to_filename(dest) rest = posixpath.normpath(request.path_info) proxied_url = dest.lstrip("/") + "/" + urllib.quote(rest.lstrip("/")) ## FIXME: handle /->/index.html filename = filename.rstrip("/") + "/" + rest.lstrip("/") if os.path.isdir(filename): if not request.path.endswith("/"): new_url = request.path + "/" if request.query_string: new_url += "?" + request.query_string resp = exc.HTTPMovedPermanently(location=new_url) return resp, orig_base, dest, proxied_url ## FIXME: configurable? StaticURLParser? for base in ["index.html", "index.htm"]: if os.path.exists(os.path.join(filename, base)): filename = os.path.join(filename, base) break else: resp = exc.HTTPNotFound("There was no index.html file in the directory") if not os.path.exists(filename): resp = exc.HTTPNotFound("The file %s could not be found" % filename) else: app = FileApp(filename) # I don't really need a copied request here, because FileApp is so simple: resp = request.get_response(app) return resp, orig_base, dest, proxied_url
def proxy_to_file(self, request, dest): """Handle local ``file:`` URLs""" orig_base = request.application_url ## FIXME: security restrictions here? assert dest.startswith('file:') if '?' in dest: dest = dest.split('?', 1)[0] filename = url_to_filename(dest) rest = posixpath.normpath(request.path_info) proxied_url = dest.lstrip('/') + '/' + urllib.quote(rest.lstrip('/')) ## FIXME: handle /->/index.html filename = filename.rstrip('/') + '/' + rest.lstrip('/') if os.path.isdir(filename): if not request.path.endswith('/'): new_url = request.path + '/' if request.query_string: new_url += '?' + request.query_string resp = exc.HTTPMovedPermanently(location=new_url) return resp, orig_base, dest, proxied_url ## FIXME: configurable? StaticURLParser? for base in ['index.html', 'index.htm']: if os.path.exists(os.path.join(filename, base)): filename = os.path.join(filename, base) break else: resp = exc.HTTPNotFound("There was no index.html file in the directory") if not os.path.exists(filename): resp = exc.HTTPNotFound("The file %s could not be found" % filename) else: app = FileApp(filename) resp = request.get_response(app) return resp, orig_base, dest, proxied_url
def get_resource(self, url, orig_req, log): """ Gets the resource at the given url, using the original request `orig_req` as the basis for constructing the subrequest. Returns a `webob.Response` object. """ assert url is not None if url.lower().startswith('file:'): if not display_local_files(orig_req): ## FIXME: not sure if this applies generally; some ## calls to get_resource might be because of a more ## valid subrequest than displaying a file return exc.HTTPForbidden( "You cannot access file: URLs (like %r)" % url) filename = url_to_filename(url) if not os.path.exists(filename): return exc.HTTPNotFound( "The file %r was not found" % filename) if os.path.isdir(filename): return exc.HTTPForbidden( "You cannot display a directory (%r)" % filename) subresp = Response() type, dummy = mimetypes.guess_type(filename) if not type: type = 'application/octet-stream' subresp.content_type = type ## FIXME: reading the whole thing obviously ain't great: f = open(filename, 'rb') subresp.body = f.read() f.close() return subresp elif url.startswith(orig_req.application_url + '/'): subreq = orig_req.copy_get() subreq.environ['deliverance.subrequest_original_environ'] = orig_req.environ #jhb subreq.environ['deliverance.subrequest_original_request'] = orig_req new_path_info = url[len(orig_req.application_url):] query_string = '' if '?' in new_path_info: new_path_info, query_string = new_path_info.split('?') new_path_info = urllib.unquote(new_path_info) assert new_path_info.startswith('/') subreq.path_info = new_path_info subreq.query_string = query_string subresp = subreq.get_response(self.app) ## FIXME: error if not HTML? ## FIXME: handle redirects? ## FIXME: handle non-200? log.debug(self, 'Internal request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) return subresp else: ## FIXME: pluggable subrequest handler? subreq = Request.blank(url) subresp = subreq.get_response(proxy_exact_request) log.debug(self, 'External request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) return subresp
def action_edit_rules(self, req, resource_fetcher): if not edit_local_files(req.environ): return exc.HTTPForbidden('Editing is forbidden') rules = self.rule_getter(resource_fetcher, self.app, req) file_url = rules.source_location if not file_url.startswith('file:'): return exc.HTTPForbidden('The rule location (%s) is not a local file' % file_url) filename = url_to_filename(file_url) app = Editor(filename=filename, force_syntax='delivxml', title='rule file %s' % os.path.basename(filename)) return app
def action_edit_rules(self, req, resource_fetcher): if not edit_local_files(req.environ): return exc.HTTPForbidden('Editing is forbidden') rules = self.rule_getter(resource_fetcher, self.app, req) file_url = rules.source_location if not file_url.startswith('file:'): return exc.HTTPForbidden( 'The rule location (%s) is not a local file' % file_url) filename = url_to_filename(file_url) app = Editor(filename=filename, force_syntax='delivxml', title='rule file %s' % os.path.basename(filename)) return app
def edit_app(self, environ, start_response): try: if not self.editable: raise exc.HTTPForbidden('This proxy is not editable="1"') if not edit_local_files(environ): raise exc.HTTPForbidden("Editing is forbidden") try: dest_href = uri_template_substitute(self.dest.href, dict(here=posixpath.dirname(self.source_location))) except KeyError: raise exc.HTTPForbidden("Not a static location: %s" % self.dest.href) if not dest_href.startswith("file:/"): raise exc.HTTPForbidden("Not local: %s" % self.dest.href) filename = url_to_filename(dest_href) editor = Editor(base_dir=filename) return editor(environ, start_response) except exc.HTTPException, e: return e(environ, start_response)
def expand_filename(filename, source_location=None): """ Expand environmental variables in a filename """ if source_location and source_location.startswith('file:'): here = os.path.dirname(filetourl.url_to_filename(source_location)) else: ## FIXME: this is a lousy default: here = '' vars = NestedDict(dict(here=here), DefaultDict(os.environ)) tmpl = Template(filename) try: return tmpl.substitute(vars) except ValueError, e: raise DeliveranceSyntaxError( "The filename %r contains bad $ substitutions: %s" % (filename, e), filename, source_location=source_location)
def expand_filename(filename, source_location=None): """ Expand environmental variables in a filename """ if source_location and source_location.startswith('file:'): here = os.path.dirname(filetourl.url_to_filename(source_location)) else: ## FIXME: this is a lousy default: here = '' vars = NestedDict(dict(here=here), DefaultDict(os.environ)) tmpl = Template(filename) try: return tmpl.substitute(vars) except ValueError as e: raise DeliveranceSyntaxError( "The filename %r contains bad $ substitutions: %s" % (filename, e), filename, source_location=source_location)
def edit_app(self, environ, start_response): try: if not self.editable: raise exc.HTTPForbidden('This proxy is not editable="1"') if not edit_local_files(environ): raise exc.HTTPForbidden('Editing is forbidden') try: dest_href = uri_template_substitute( self.dest.href, dict(here=posixpath.dirname(self.source_location))) except KeyError: raise exc.HTTPForbidden('Not a static location: %s' % self.dest.href) if not dest_href.startswith('file:/'): raise exc.HTTPForbidden('Not local: %s' % self.dest.href) filename = url_to_filename(dest_href) editor = Editor(base_dir=filename) return editor(environ, start_response) except exc.HTTPException, e: return e(environ, start_response)
def parse_xml(cls, el, source_location, environ=None, traverse=False): """Parse an instance from an etree XML element""" middleware_factory = None middleware_factory_kwargs = None if traverse and el.tag != 'server-settings': try: el = el.xpath('//server-settings')[0] except IndexError: raise DeliveranceSyntaxError( "There is no <server-settings> element", element=el) if environ is None: environ = os.environ assert el.tag == 'server-settings' server_host = 'localhost:8080' ## FIXME: should these defaults be passed in: execute_pyref = True display_local_files = True edit_local_files = True dev_allow_ips = [] dev_deny_ips = [] dev_htpasswd = None dev_expiration = 0 dev_users = {} dev_secret_file = os.path.join(tempfile.gettempdir(), 'deliverance', 'devauth.txt') for child in el: if child.tag is Comment: continue ## FIXME: should some of these be attributes? elif child.tag == 'server': server_host = cls.substitute(child.text, environ) elif child.tag == 'execute-pyref': execute_pyref = asbool(cls.substitute(child.text, environ)) elif child.tag == 'dev-allow': dev_allow_ips.extend( cls.substitute(child.text, environ).split()) elif child.tag == 'dev-deny': dev_deny_ips.extend( cls.substitute(child.text, environ).split()) elif child.tag == 'dev-htpasswd': dev_htpasswd = os.path.join( os.path.dirname(url_to_filename(source_location)), cls.substitute(child.text, environ)) elif child.tag == 'dev-expiration': dev_expiration = cls.substitute(child.text, environ) if dev_expiration: dev_expiration = int(dev_expiration) elif child.tag == 'display-local-files': display_local_files = asbool( cls.substitute(child.text, environ)) elif child.tag == 'edit-local-files': edit_local_files = asbool(cls.substitute(child.text, environ)) elif child.tag == 'dev-user': username = cls.substitute(child.get('username', ''), environ) ## FIXME: allow hashed password? password = cls.substitute(child.get('password', ''), environ) if not username or not password: raise DeliveranceSyntaxError( "<dev-user> must have both a username and password attribute", element=child) if username in dev_users: raise DeliveranceSyntaxError( '<dev-user username="******"> appears more than once' % username, element=el) dev_users[username] = password elif child.tag == 'dev-secret-file': dev_secret_file = cls.substitute(child.text, environ) elif child.tag == 'middleware-factory': ref = PyReference.parse_xml(child, source_location) middleware_factory = ref.function middleware_factory_kwargs = ref.args or None else: raise DeliveranceSyntaxError( 'Unknown element in <server-settings>: <%s>' % child.tag, element=child) if dev_users and dev_htpasswd: raise DeliveranceSyntaxError( "You can use <dev-htpasswd> or <dev-user>, but not both", element=el) if not dev_users and not dev_htpasswd: ## FIXME: not sure this is the best way to warn print( 'Warning: no <dev-users> or <dev-htpasswd>; logging is inaccessible' ) ## FIXME: add a default allow_ips of 127.0.0.1? return cls(server_host, execute_pyref=execute_pyref, display_local_files=display_local_files, edit_local_files=edit_local_files, dev_allow_ips=dev_allow_ips, dev_deny_ips=dev_deny_ips, dev_users=dev_users, dev_htpasswd=dev_htpasswd, dev_expiration=dev_expiration, source_location=source_location, dev_secret_file=dev_secret_file, middleware_factory=middleware_factory, middleware_factory_kwargs=middleware_factory_kwargs)
def parse_xml(cls, el, source_location, environ=None, traverse=False): """Parse an instance from an etree XML element""" if traverse and el.tag != "server-settings": try: el = el.xpath("//server-settings")[0] except IndexError: raise DeliveranceSyntaxError("There is no <server-settings> element", element=el) if environ is None: environ = os.environ assert el.tag == "server-settings" server_host = "localhost:8080" ## FIXME: should these defaults be passed in: execute_pyref = True display_local_files = True edit_local_files = True dev_allow_ips = [] dev_deny_ips = [] dev_htpasswd = None dev_expiration = 0 dev_users = {} dev_secret_file = os.path.join(tempfile.gettempdir(), "deliverance", "devauth.txt") for child in el: if child.tag is Comment: continue ## FIXME: should some of these be attributes? elif child.tag == "server": server_host = cls.substitute(child.text, environ) elif child.tag == "execute-pyref": execute_pyref = asbool(cls.substitute(child.text, environ)) elif child.tag == "dev-allow": dev_allow_ips.extend(cls.substitute(child.text, environ).split()) elif child.tag == "dev-deny": dev_deny_ips.extend(cls.substitute(child.text, environ).split()) elif child.tag == "dev-htpasswd": dev_htpasswd = os.path.join( os.path.dirname(url_to_filename(source_location)), cls.substitute(child.text, environ) ) elif child.tag == "dev-expiration": dev_expiration = cls.substitute(child.text, environ) if dev_expiration: dev_expiration = int(dev_expiration) elif child.tag == "display-local-files": display_local_files = asbool(cls.substitute(child.text, environ)) elif child.tag == "edit-local-files": edit_local_files = asbool(cls.substitute(child.text, environ)) elif child.tag == "dev-user": username = cls.substitute(child.get("username", ""), environ) ## FIXME: allow hashed password? password = cls.substitute(child.get("password", ""), environ) if not username or not password: raise DeliveranceSyntaxError( "<dev-user> must have both a username and password attribute", element=child ) if username in dev_users: raise DeliveranceSyntaxError( '<dev-user username="******"> appears more than once' % username, element=el ) dev_users[username] = password elif child.tag == "dev-secret-file": dev_secret_file = cls.substitute(child.text, environ) else: raise DeliveranceSyntaxError("Unknown element in <server-settings>: <%s>" % child.tag, element=child) if dev_users and dev_htpasswd: raise DeliveranceSyntaxError("You can use <dev-htpasswd> or <dev-user>, but not both", element=el) if not dev_users and not dev_htpasswd: ## FIXME: not sure this is the best way to warn print "Warning: no <dev-users> or <dev-htpasswd>; logging is inaccessible" ## FIXME: add a default allow_ips of 127.0.0.1? return cls( server_host, execute_pyref=execute_pyref, display_local_files=display_local_files, edit_local_files=edit_local_files, dev_allow_ips=dev_allow_ips, dev_deny_ips=dev_deny_ips, dev_users=dev_users, dev_htpasswd=dev_htpasswd, dev_expiration=dev_expiration, source_location=source_location, dev_secret_file=dev_secret_file, )
def get_resource(self, url, orig_req, log, retry_inner_if_not_200=False): """ Gets the resource at the given url, using the original request `orig_req` as the basis for constructing the subrequest. Returns a `webob.Response` object. If `url.startswith(orig_req.application_url + '/')`, then Deliverance will try to fetch the resource by making a subrequest to the app that is being wrapped by Deliverance, instead of an external subrequest. This can cause problems in some setups -- see #16. To work around this, if `retry_inner_if_not_200` is True, then, in the situation described above, non-200 responses from the inner app will be tossed out, and the request will be retried as an external http request. Currently this is used only by RuleSet.get_theme """ assert url is not None force_external = 'deliv_force_external' in urlparse.urlsplit(url).query if url.lower().startswith('file:'): if not display_local_files(orig_req): ## FIXME: not sure if this applies generally; some ## calls to get_resource might be because of a more ## valid subrequest than displaying a file return exc.HTTPForbidden( "You cannot access file: URLs (like %r)" % url) filename = url_to_filename(url) if not os.path.exists(filename): return exc.HTTPNotFound("The file %r was not found" % filename) if os.path.isdir(filename): return exc.HTTPForbidden( "You cannot display a directory (%r)" % filename) subresp = Response() type, dummy = mimetypes.guess_type(filename) if not type: type = 'application/octet-stream' subresp.content_type = type ## FIXME: reading the whole thing obviously ain't great: f = open(filename, 'rb') subresp.body = f.read() f.close() return subresp elif not force_external and\ url.startswith(orig_req.application_url + '/'): subreq = orig_req.copy_get() subreq.environ[ 'deliverance.subrequest_original_environ'] = orig_req.environ new_path_info = url[len(orig_req.application_url):] query_string = '' if '?' in new_path_info: new_path_info, query_string = new_path_info.split('?', 1) new_path_info = urllib.unquote(new_path_info) assert new_path_info.startswith('/') subreq.path_info = new_path_info subreq.query_string = query_string subresp = subreq.get_response(self.app) ## FIXME: error if not HTML? ## FIXME: handle redirects? ## FIXME: handle non-200? log.debug(self, 'Internal request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) if not retry_inner_if_not_200: return subresp if subresp.status_int == 200: return subresp elif 'x-deliverance-theme-subrequest' in orig_req.headers: log.debug( self, 'Internal request for %s was not 200 OK; ' 'returning it anyway.' % url) return subresp else: log.debug( self, 'Internal request for %s was not 200 OK; retrying as external request.' % url) ## FIXME: ZCA aptaperized subrequest handler subreq = self.build_external_subrequest(url, orig_req, log) subresp = subreq.get_response(proxy_exact_request) log.debug(self, 'External request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) return subresp
def get_resource(self, url, orig_req, log, retry_inner_if_not_200=False): """ Gets the resource at the given url, using the original request `orig_req` as the basis for constructing the subrequest. Returns a `webob.Response` object. If `url.startswith(orig_req.application_url + '/')`, then Deliverance will try to fetch the resource by making a subrequest to the app that is being wrapped by Deliverance, instead of an external subrequest. This can cause problems in some setups -- see #16. To work around this, if `retry_inner_if_not_200` is True, then, in the situation described above, non-200 responses from the inner app will be tossed out, and the request will be retried as an external http request. Currently this is used only by RuleSet.get_theme """ assert url is not None if url.lower().startswith('file:'): if not display_local_files(orig_req): ## FIXME: not sure if this applies generally; some ## calls to get_resource might be because of a more ## valid subrequest than displaying a file return exc.HTTPForbidden( "You cannot access file: URLs (like %r)" % url) filename = url_to_filename(url) if not os.path.exists(filename): return exc.HTTPNotFound( "The file %r was not found" % filename) if os.path.isdir(filename): return exc.HTTPForbidden( "You cannot display a directory (%r)" % filename) subresp = Response() type, dummy = mimetypes.guess_type(filename) if not type: type = 'application/octet-stream' subresp.content_type = type ## FIXME: reading the whole thing obviously ain't great: f = open(filename, 'rb') subresp.body = f.read() f.close() return subresp elif self.use_internal_subrequest(url, orig_req, log): subreq = orig_req.copy_get() subreq.environ['deliverance.subrequest_original_environ'] = orig_req.environ new_path_info = url[len(orig_req.application_url):] query_string = '' if '?' in new_path_info: new_path_info, query_string = new_path_info.split('?', 1) new_path_info = urllib.unquote(new_path_info) assert new_path_info.startswith('/') subreq.path_info = new_path_info subreq.query_string = query_string subresp = subreq.get_response(self.app) ## FIXME: error if not HTML? ## FIXME: handle redirects? ## FIXME: handle non-200? log.debug(self, 'Internal request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) if not retry_inner_if_not_200: return subresp if subresp.status_int == 200: return subresp elif 'x-deliverance-theme-subrequest' in orig_req.headers: log.debug(self, 'Internal request for %s was not 200 OK; ' 'returning it anyway.' % url) return subresp else: log.debug(self, 'Internal request for %s was not 200 OK; retrying as external request.' % url) ## FIXME: pluggable subrequest handler? subreq = self.build_external_subrequest(url, orig_req, log) subresp = subreq.get_response(proxy_exact_request) log.debug(self, 'External request for %s: %s content-type: %s', url, subresp.status, subresp.content_type) return subresp
def parse_xml(cls, el, source_location, environ=None, traverse=False): """Parse an instance from an etree XML element""" middleware_factory = None middleware_factory_kwargs = None if traverse and el.tag != 'server-settings': try: el = el.xpath('//server-settings')[0] except IndexError: raise DeliveranceSyntaxError( "There is no <server-settings> element", element=el) if environ is None: environ = os.environ assert el.tag == 'server-settings' server_host = 'localhost:8080' ## FIXME: should these defaults be passed in: execute_pyref = True display_local_files = True edit_local_files = True dev_allow_ips = [] dev_deny_ips = [] dev_htpasswd = None dev_expiration = 0 dev_users = {} dev_secret_file = os.path.join(tempfile.gettempdir(), 'deliverance', 'devauth.txt') for child in el: if child.tag is Comment: continue ## FIXME: should some of these be attributes? elif child.tag == 'server': server_host = cls.substitute(child.text, environ) elif child.tag == 'execute-pyref': execute_pyref = asbool(cls.substitute(child.text, environ)) elif child.tag == 'dev-allow': dev_allow_ips.extend(cls.substitute(child.text, environ).split()) elif child.tag == 'dev-deny': dev_deny_ips.extend(cls.substitute(child.text, environ).split()) elif child.tag == 'dev-htpasswd': dev_htpasswd = os.path.join(os.path.dirname(url_to_filename(source_location)), cls.substitute(child.text, environ)) elif child.tag == 'dev-expiration': dev_expiration = cls.substitute(child.text, environ) if dev_expiration: dev_expiration = int(dev_expiration) elif child.tag == 'display-local-files': display_local_files = asbool(cls.substitute(child.text, environ)) elif child.tag == 'edit-local-files': edit_local_files = asbool(cls.substitute(child.text, environ)) elif child.tag == 'dev-user': username = cls.substitute(child.get('username', ''), environ) ## FIXME: allow hashed password? password = cls.substitute(child.get('password', ''), environ) if not username or not password: raise DeliveranceSyntaxError( "<dev-user> must have both a username and password attribute", element=child) if username in dev_users: raise DeliveranceSyntaxError( '<dev-user username="******"> appears more than once' % username, element=el) dev_users[username] = password elif child.tag == 'dev-secret-file': dev_secret_file = cls.substitute(child.text, environ) elif child.tag == 'middleware-factory': ref = PyReference.parse_xml(child, source_location) middleware_factory = ref.function middleware_factory_kwargs = ref.args or None else: raise DeliveranceSyntaxError( 'Unknown element in <server-settings>: <%s>' % child.tag, element=child) if dev_users and dev_htpasswd: raise DeliveranceSyntaxError( "You can use <dev-htpasswd> or <dev-user>, but not both", element=el) if not dev_users and not dev_htpasswd: ## FIXME: not sure this is the best way to warn print 'Warning: no <dev-users> or <dev-htpasswd>; logging is inaccessible' ## FIXME: add a default allow_ips of 127.0.0.1? return cls(server_host, execute_pyref=execute_pyref, display_local_files=display_local_files, edit_local_files=edit_local_files, dev_allow_ips=dev_allow_ips, dev_deny_ips=dev_deny_ips, dev_users=dev_users, dev_htpasswd=dev_htpasswd, dev_expiration=dev_expiration, source_location=source_location, dev_secret_file=dev_secret_file, middleware_factory=middleware_factory, middleware_factory_kwargs=middleware_factory_kwargs)