def register_loader(self, loader, prefix=None, replace=False): if not isinstance(self.environment.loader, MacroLoader): raise Exception("The macro system requires the Jinja loader to be wrapped using an instance of MacroLoader") if prefix: loader = PrefixLoader(dict([(prefix, loader)])) self.environment.loader.macro_loaders.append(loader) for tpl in loader.list_templates(): self.register_from_template("__macros__/" + tpl, replace)
def initialize(self, *args, **kwargs): """Load configuration settings.""" super().initialize(*args, **kwargs) self.load_config_file(self.config_file) # hook up tornado logging if self.debug: self.log_level = logging.DEBUG tornado.options.options.logging = logging.getLevelName(self.log_level) tornado.log.enable_pretty_logging() self.log = tornado.log.app_log self.init_pycurl() # initialize kubernetes config if self.builder_required: try: kubernetes.config.load_incluster_config() except kubernetes.config.ConfigException: kubernetes.config.load_kube_config() self.tornado_settings["kubernetes_client"] = self.kube_client = kubernetes.client.CoreV1Api() # times 2 for log + build threads self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2) # default executor for asyncifying blocking calls (e.g. to kubernetes, docker). # this should not be used for long-running requests self.executor = ThreadPoolExecutor(self.executor_threads) jinja_options = dict(autoescape=True, ) template_paths = [self.template_path] base_template_path = self._template_path_default() if base_template_path not in template_paths: # add base templates to the end, so they are looked up at last after custom templates template_paths.append(base_template_path) loader = ChoiceLoader([ # first load base templates with prefix PrefixLoader({'templates': FileSystemLoader([base_template_path])}, '/'), # load all templates FileSystemLoader(template_paths) ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry and self.builder_required: registry = DockerRegistry(parent=self) else: registry = None self.launcher = Launcher( parent=self, hub_url=self.hub_url, hub_api_token=self.hub_api_token, create_user=not self.auth_enabled, ) self.event_log = EventLog(parent=self) for schema_file in glob(os.path.join(HERE, 'event-schemas','*.json')): with open(schema_file) as f: self.event_log.register_schema(json.load(f)) self.tornado_settings.update({ "push_secret": self.push_secret, "image_prefix": self.image_prefix, "debug": self.debug, 'launcher': self.launcher, 'appendix': self.appendix, "build_namespace": self.build_namespace, "build_image": self.build_image, 'build_node_selector': self.build_node_selector, 'build_pool': self.build_pool, 'log_tail_lines': self.log_tail_lines, 'per_repo_quota': self.per_repo_quota, 'repo_providers': self.repo_providers, 'use_registry': self.use_registry, 'registry': registry, 'traitlets_config': self.config, 'google_analytics_code': self.google_analytics_code, 'google_analytics_domain': self.google_analytics_domain, 'about_message': self.about_message, 'extra_footer_scripts': self.extra_footer_scripts, 'jinja2_env': jinja_env, 'build_memory_limit': self.build_memory_limit, 'build_docker_host': self.build_docker_host, 'base_url': self.base_url, "static_path": os.path.join(HERE, "static"), 'static_url_prefix': url_path_join(self.base_url, 'static/'), 'template_variables': self.template_variables, 'executor': self.executor, 'auth_enabled': self.auth_enabled, 'use_named_servers': self.use_named_servers, 'event_log': self.event_log }) if self.auth_enabled: self.tornado_settings['cookie_secret'] = os.urandom(32) handlers = [ (r'/metrics', MetricsHandler), (r"/build/([^/]+)/(.+)", BuildHandler), (r"/v2/([^/]+)/(.+)", ParameterizedMainHandler), (r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler), # for backward-compatible mybinder.org badge URLs # /assets/images/badge.svg (r'/assets/(images/badge\.svg)', tornado.web.StaticFileHandler, {'path': self.tornado_settings['static_path']}), # /badge.svg (r'/(badge\.svg)', tornado.web.StaticFileHandler, {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), # /badge_logo.svg (r'/(badge\_logo\.svg)', tornado.web.StaticFileHandler, {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), # /favicon_XXX.ico (r'/(favicon\_fail\.ico)', tornado.web.StaticFileHandler, {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), (r'/(favicon\_success\.ico)', tornado.web.StaticFileHandler, {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), (r'/(favicon\_building\.ico)', tornado.web.StaticFileHandler, {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), (r'/about', AboutHandler), (r'/', MainHandler), (r'.*', Custom404), ] handlers = self.add_url_prefix(self.base_url, handlers) if self.extra_static_path: handlers.insert(-1, (re.escape(url_path_join(self.base_url, self.extra_static_url_prefix)) + r"(.*)", tornado.web.StaticFileHandler, {'path': self.extra_static_path})) if self.auth_enabled: oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \ url_path_join(self.base_url, 'oauth_callback') oauth_redirect_uri = urlparse(oauth_redirect_uri).path handlers.insert(-1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler)) self.tornado_app = tornado.web.Application(handlers, **self.tornado_settings)
def initialize(self, *args, **kwargs): """Load configuration settings.""" super().initialize(*args, **kwargs) self.load_config_file(self.config_file) # hook up tornado logging if self.debug: self.log_level = logging.DEBUG tornado.options.options.logging = logging.getLevelName(self.log_level) tornado.log.enable_pretty_logging() self.log = tornado.log.app_log self.init_pycurl() # initialize kubernetes config if self.builder_required: try: kubernetes.config.load_incluster_config() except kubernetes.config.ConfigException: kubernetes.config.load_kube_config() self.tornado_settings[ "kubernetes_client"] = self.kube_client = kubernetes.client.CoreV1Api( ) # times 2 for log + build threads self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2) # default executor for asyncifying blocking calls (e.g. to kubernetes, docker). # this should not be used for long-running requests self.executor = ThreadPoolExecutor(self.executor_threads) jinja_options = dict(autoescape=True, ) template_paths = [self.template_path] base_template_path = self._template_path_default() if base_template_path not in template_paths: # add base templates to the end, so they are looked up at last after custom templates template_paths.append(base_template_path) loader = ChoiceLoader([ # first load base templates with prefix PrefixLoader({'templates': FileSystemLoader([base_template_path])}, '/'), # load all templates FileSystemLoader(template_paths) ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry: registry = self.registry_class(parent=self) else: registry = None self.launcher = Launcher( parent=self, hub_url=self.hub_url, hub_url_local=self.hub_url_local, hub_api_token=self.hub_api_token, create_user=not self.auth_enabled, ) self.event_log = EventLog(parent=self) for schema_file in glob(os.path.join(HERE, 'event-schemas', '*.json')): with open(schema_file) as f: self.event_log.register_schema(json.load(f)) self.tornado_settings.update({ "log_function": log_request, "push_secret": self.push_secret, "image_prefix": self.image_prefix, "debug": self.debug, "hub_url": self.hub_url, "launcher": self.launcher, "appendix": self.appendix, "ban_networks": self.ban_networks, "ban_networks_min_prefix_len": self.ban_networks_min_prefix_len, "build_namespace": self.build_namespace, "build_image": self.build_image, "build_node_selector": self.build_node_selector, "build_pool": self.build_pool, "build_token_check_origin": self.build_token_check_origin, "build_token_secret": self.build_token_secret, "build_token_expires_seconds": self.build_token_expires_seconds, "sticky_builds": self.sticky_builds, "log_tail_lines": self.log_tail_lines, "pod_quota": self.pod_quota, "per_repo_quota": self.per_repo_quota, "per_repo_quota_higher": self.per_repo_quota_higher, "repo_providers": self.repo_providers, "rate_limiter": RateLimiter(parent=self), "use_registry": self.use_registry, "build_class": self.build_class, "registry": registry, "traitlets_config": self.config, "google_analytics_code": self.google_analytics_code, "google_analytics_domain": self.google_analytics_domain, "about_message": self.about_message, "banner_message": self.banner_message, "extra_footer_scripts": self.extra_footer_scripts, "jinja2_env": jinja_env, "build_memory_limit": self.build_memory_limit, "build_memory_request": self.build_memory_request, "build_docker_host": self.build_docker_host, "build_docker_config": self.build_docker_config, "base_url": self.base_url, "badge_base_url": self.badge_base_url, "static_path": os.path.join(HERE, "static"), "static_url_prefix": url_path_join(self.base_url, "static/"), "template_variables": self.template_variables, "executor": self.executor, "auth_enabled": self.auth_enabled, "event_log": self.event_log, "normalized_origin": self.normalized_origin, "allowed_metrics_ips": set(map(ipaddress.ip_network, self.allowed_metrics_ips)) }) if self.auth_enabled: self.tornado_settings['cookie_secret'] = os.urandom(32) if self.cors_allow_origin: self.tornado_settings.setdefault( 'headers', {})['Access-Control-Allow-Origin'] = self.cors_allow_origin handlers = [ (r'/metrics', MetricsHandler), (r'/versions', VersionHandler), (r"/build/([^/]+)/(.+)", BuildHandler), (r"/v2/([^/]+)/(.+)", ParameterizedMainHandler), (r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler), (r'/~([^/]+/.*)', UserRedirectHandler), # for backward-compatible mybinder.org badge URLs # /assets/images/badge.svg (r'/assets/(images/badge\.svg)', tornado.web.StaticFileHandler, { 'path': self.tornado_settings['static_path'] }), # /badge.svg (r'/(badge\.svg)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), # /badge_logo.svg (r'/(badge\_logo\.svg)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), # /logo_social.png (r'/(logo\_social\.png)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), # /favicon_XXX.ico (r'/(favicon\_fail\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/(favicon\_success\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/(favicon\_building\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/about', AboutHandler), (r'/health', HealthHandler, { 'hub_url': self.hub_url_local }), (r'/_config', ConfigHandler), (r'/', MainHandler), (r'.*', Custom404), ] handlers = self.add_url_prefix(self.base_url, handlers) if self.extra_static_path: handlers.insert(-1, (re.escape( url_path_join(self.base_url, self.extra_static_url_prefix)) + r"(.*)", tornado.web.StaticFileHandler, { 'path': self.extra_static_path })) if self.auth_enabled: oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \ url_path_join(self.base_url, 'oauth_callback') oauth_redirect_uri = urlparse(oauth_redirect_uri).path handlers.insert( -1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler)) self.tornado_app = tornado.web.Application(handlers, **self.tornado_settings)
def app(self): "Returns an ASGI app function that serves the whole of Datasette" default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) template_paths.extend([ plugin["templates_path"] for plugin in get_plugins(pm) if plugin["templates_path"] ]) template_paths.append(default_templates) template_loader = ChoiceLoader([ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader({"default": FileSystemLoader(default_templates)}, delimiter=":"), ]) self.jinja_env = Environment(loader=template_loader, autoescape=True, enable_async=True) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters[ "quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) self.register_renderers() routes = [] def add_route(view, regex): routes.append((regex, view)) # Generate a regex snippet to match all registered renderer file extensions renderer_regex = "|".join(r"\." + key for key in self.renderers.keys()) add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires add_route(favicon, "/favicon.ico") add_route(asgi_static(app_root / "datasette" / "static"), r"/-/static/(?P<path>.*)$") for path, dirname in self.static_mounts: add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$") # Mount any plugin static/ directories for plugin in get_plugins(pm): if plugin["static_path"]: add_route( asgi_static(plugin["static_path"]), "/-/static-plugins/{}/(?P<path>.*)$".format( plugin["name"]), ) # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611 add_route( asgi_static(plugin["static_path"]), "/-/static-plugins/{}/(?P<path>.*)$".format( plugin["name"].replace("-", "_")), ) add_route( JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata), r"/-/metadata(?P<as_format>(\.json)?)$", ) add_route( JsonDataView.as_asgi(self, "versions.json", self.versions), r"/-/versions(?P<as_format>(\.json)?)$", ) add_route( JsonDataView.as_asgi(self, "plugins.json", self.plugins), r"/-/plugins(?P<as_format>(\.json)?)$", ) add_route( JsonDataView.as_asgi(self, "config.json", lambda: self._config), r"/-/config(?P<as_format>(\.json)?)$", ) add_route( JsonDataView.as_asgi(self, "threads.json", self.threads), r"/-/threads(?P<as_format>(\.json)?)$", ) add_route( JsonDataView.as_asgi(self, "databases.json", self.connected_databases), r"/-/databases(?P<as_format>(\.json)?)$", ) add_route(DatabaseDownload.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$") add_route( DatabaseView.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_format>" + renderer_regex + r"|.jsono|\.csv)?$", ) add_route( TableView.as_asgi(self), r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", ) add_route( RowView.as_asgi(self), r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>" + renderer_regex + r")?$", ) self.register_custom_units() async def setup_db(): # First time server starts up, calculate table counts for immutable databases for dbname, database in self.databases.items(): if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) asgi = AsgiLifespan(AsgiTracer(DatasetteRouter(self, routes)), on_startup=setup_db) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) return asgi
import os from jinja2 import Environment from jinja2 import PackageLoader from jinja2 import PrefixLoader from . import conf from . import util __template_packages = {'default': PackageLoader('redreport', 'templates')} for plugin in conf['plugins']: try: module = __import__(plugin) except ImportError: pass else: if os.path.exists(os.path.join(module.__path__), 'templates'): __template_packages[plugin] = PackageLoader(plugin, 'templates') env = Environment(loader=PrefixLoader(__template_packages)) env.filters['dateformat'] = util.dateformat env.filters['monthformat'] = util.monthformat
from lender_ui.landregistry_flask import LandRegistryFlask from jinja2 import PackageLoader from jinja2 import PrefixLoader app = LandRegistryFlask( __name__, # template_folder='templates', static_folder='assets/dist', static_url_path='/ui') # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = PrefixLoader({'app': PackageLoader('lender_ui')}) app.config.from_pyfile("config.py") @app.context_processor def inject_global_values(): """Inject global template values Use this to inject values into the templates that are used globally. This might be things such as google analytics keys, or even the current username """ return dict(service_name='Lender UI')
def app(self): class TracingSanic(Sanic): async def handle_request(self, request, write_callback, stream_callback): if request.args.get("_trace"): request["traces"] = [] request["trace_start"] = time.time() with capture_traces(request["traces"]): await super().handle_request(request, write_callback, stream_callback) else: await super().handle_request(request, write_callback, stream_callback) app = TracingSanic(__name__) default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) template_paths.extend([ plugin["templates_path"] for plugin in get_plugins(pm) if plugin["templates_path"] ]) template_paths.append(default_templates) template_loader = ChoiceLoader([ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader({"default": FileSystemLoader(default_templates)}, delimiter=":"), ]) self.jinja_env = Environment(loader=template_loader, autoescape=True) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters[ "quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) self.register_renderers() # Generate a regex snippet to match all registered renderer file extensions renderer_regex = "|".join(r"\." + key for key in self.renderers.keys()) app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires app.add_route(favicon, "/favicon.ico") app.static("/-/static/", str(app_root / "datasette" / "static")) for path, dirname in self.static_mounts: app.static(path, dirname) # Mount any plugin static/ directories for plugin in get_plugins(pm): if plugin["static_path"]: modpath = "/-/static-plugins/{}/".format(plugin["name"]) app.static(modpath, plugin["static_path"]) app.add_route( JsonDataView.as_view(self, "metadata.json", lambda: self._metadata), r"/-/metadata<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "versions.json", self.versions), r"/-/versions<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "plugins.json", self.plugins), r"/-/plugins<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "config.json", lambda: self._config), r"/-/config<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "databases.json", self.connected_databases), r"/-/databases<as_format:(\.json)?$>", ) app.add_route(DatabaseDownload.as_view(self), r"/<db_name:[^/]+?><as_db:(\.db)$>") app.add_route( DatabaseView.as_view(self), r"/<db_name:[^/]+?><as_format:(" + renderer_regex + r"|.jsono|\.csv)?$>", ) app.add_route(TableView.as_view(self), r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>") app.add_route( RowView.as_view(self), r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(" + renderer_regex + r")?$>", ) self.register_custom_units() # On 404 with a trailing slash redirect to path without that slash: # pylint: disable=unused-variable @app.middleware("response") def redirect_on_404_with_trailing_slash(request, original_response): if original_response.status == 404 and request.path.endswith("/"): path = request.path.rstrip("/") if request.query_string: path = "{}?{}".format(path, request.query_string) return response.redirect(path) @app.middleware("response") async def add_traces_to_response(request, response): if request.get("traces") is None: return traces = request["traces"] trace_info = { "request_duration_ms": 1000 * (time.time() - request["trace_start"]), "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces), "num_traces": len(traces), "traces": traces, } if "text/html" in response.content_type and b"</body>" in response.body: extra = json.dumps(trace_info, indent=2) extra_html = "<pre>{}</pre></body>".format(extra).encode( "utf8") response.body = response.body.replace(b"</body>", extra_html) elif "json" in response.content_type and response.body.startswith( b"{"): data = json.loads(response.body.decode("utf8")) if "_trace" not in data: data["_trace"] = trace_info response.body = json.dumps(data).encode("utf8") @app.exception(Exception) def on_exception(request, exception): title = None help = None if isinstance(exception, NotFound): status = 404 info = {} message = exception.args[0] elif isinstance(exception, InvalidUsage): status = 405 info = {} message = exception.args[0] elif isinstance(exception, DatasetteError): status = exception.status info = exception.error_dict message = exception.message if exception.messagge_is_html: message = Markup(message) title = exception.title else: status = 500 info = {} message = str(exception) traceback.print_exc() templates = ["500.html"] if status != 500: templates = ["{}.html".format(status)] + templates info.update({ "ok": False, "error": message, "status": status, "title": title }) if request is not None and request.path.split("?")[0].endswith( ".json"): r = response.json(info, status=status) else: template = self.jinja_env.select_template(templates) r = response.html(template.render(info), status=status) if self.cors: r.headers["Access-Control-Allow-Origin"] = "*" return r # First time server starts up, calculate table counts for immutable databases @app.listener("before_server_start") async def setup_db(app, loop): for dbname, database in self.databases.items(): if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) return app
#!/usr/bin/env python3 import argparse import json import os from jinja2 import Environment, FileSystemLoader, PrefixLoader loader = PrefixLoader({ 'govuk_frontend_jinja': FileSystemLoader(searchpath=os.path.join(os.path.dirname(__file__), '../../govuk_frontend_jinja/templates')) }) env = Environment(loader=loader, autoescape=True) parser = argparse.ArgumentParser(description='Render a govuk_frontend_jinja template.') parser.add_argument('--component', metavar='Component name', nargs=1, help='The name of the component') parser.add_argument('--params', metavar='Params', nargs=1, help='JSON representation of the params to pass to the component macro') args = parser.parse_args() def hyphenated_to_camel(string): return ''.join(x.capitalize() for x in string.split('-')) template = env.from_string(''' {{% from "govuk_frontend_jinja/components/" + component + "/macro.html" import govuk{macro_name} %}} {{{{ govuk{macro_name}(params) }}}} '''.format(macro_name=hyphenated_to_camel(args.component[0]))) print(template.render(
from demo.landregistry_flask import LandRegistryFlask from jinja2 import PackageLoader, FileSystemLoader, PrefixLoader app = LandRegistryFlask(__name__, template_folder='templates', static_folder='assets/dist', static_url_path='/ui') # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = PrefixLoader({ 'app': PackageLoader('demo'), 'components': FileSystemLoader('hmlr_design_system/components'), 'wtforms_gov': PackageLoader('demo.custom_extensions.wtforms_helpers') }) app.config.from_pyfile("config.py") @app.context_processor def inject_global_values(): """Inject global template values Use this to inject values into the templates that are used globally. This might be things such as google analytics keys, or even the current username """ return dict(service_name='HMLR Design System')
from title_ui.landregistry_flask import LandRegistryFlask from jinja2 import PackageLoader from jinja2 import PrefixLoader app = LandRegistryFlask(__name__, template_folder='templates', static_folder='assets/dist', static_url_path='/ui') # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = PrefixLoader({'app': PackageLoader('title_ui')}) app.config.from_pyfile("config.py") @app.context_processor def inject_global_values(): """Inject global template values Use this to inject values into the templates that are used globally. This might be things such as google analytics keys, or even the current username """ return dict(service_name='View your title')
from demo.landregistry_flask import LandRegistryFlask app = LandRegistryFlask( __name__, template_folder="templates", static_folder="assets/dist", static_url_path="/ui", ) # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = ChoiceLoader([ PackageLoader("demo"), PrefixLoader({ "components": FileSystemLoader("hmlr_design_system/components"), "govuk_frontend_jinja": PackageLoader("govuk_frontend_jinja"), "govuk_frontend_wtf": PackageLoader("govuk_frontend_wtf"), }), ]) app.config.from_pyfile("config.py") def parse_path(demo_path): demo_path = path.relpath(demo_path, "hmlr_design_system/components").replace( "/demos", "").replace(".html", "") path_parts = demo_path.split("/") return url_for( "components.component_demo",
from jinja2 import PrefixLoader from os import getcwd from os.path import join import os app = LandRegistryFlask(__name__, template_folder='templates', static_folder='assets/dist', static_url_path='/static') # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = PrefixLoader({ 'app': PackageLoader('demo.demo'), 'land_registry_elements': FileSystemLoader('src/land_registry_elements'), 'govuk_elements_jinja_macros': PackageLoader('govuk_elements_jinja_macros'), 'incubation_area': FileSystemLoader('src/incubation_area') }) app.config.from_pyfile("config.py") @app.context_processor def inject_global_values(): """Inject global template values Use this to inject values into the templates that are used globally. This might be things such as google analytics keys, or even the current username """
def __init__( self, files, immutables=None, cache_headers=True, cors=False, inspect_data=None, metadata=None, sqlite_extensions=None, template_dir=None, plugins_dir=None, static_mounts=None, memory=False, config=None, version_note=None, ): immutables = immutables or [] self.files = tuple(files) + tuple(immutables) self.immutables = set(immutables) if not self.files: self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() self.inspect_data = inspect_data for file in self.files: path = file is_memory = False if file is MEMORY: path = None is_memory = True is_mutable = path not in self.immutables db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) if db.name in self.databases: raise Exception("Multiple files with same stem: {}".format(db.name)) self.add_database(db.name, db) self.cache_headers = cache_headers self.cors = cors self._metadata = metadata or {} self.sqlite_functions = [] self.sqlite_extensions = sqlite_extensions or [] self.template_dir = template_dir self.plugins_dir = plugins_dir self.static_mounts = static_mounts or [] self._config = dict(DEFAULT_CONFIG, **(config or {})) self.renderers = {} # File extension -> renderer function self.version_note = version_note self.executor = futures.ThreadPoolExecutor( max_workers=self.config("num_sql_threads") ) self.max_returned_rows = self.config("max_returned_rows") self.sql_time_limit_ms = self.config("sql_time_limit_ms") self.page_size = self.config("default_page_size") # Execute plugins in constructor, to ensure they are available # when the rest of `datasette inspect` executes if self.plugins_dir: for filename in os.listdir(self.plugins_dir): filepath = os.path.join(self.plugins_dir, filename) mod = module_from_path(filepath, name=filename) try: pm.register(mod) except ValueError: # Plugin already registered pass # Configure Jinja default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) plugin_template_paths = [ plugin["templates_path"] for plugin in get_plugins() if plugin["templates_path"] ] template_paths.extend(plugin_template_paths) template_paths.append(default_templates) template_loader = ChoiceLoader( [ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader( {"default": FileSystemLoader(default_templates)}, delimiter=":" ), ] ) self.jinja_env = Environment( loader=template_loader, autoescape=True, enable_async=True ) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) self.register_renderers()
def setup_handlers(web_app: "NotebookWebApplication", config: JupyterProject, logger: logging.Logger): host_pattern = ".*$" base_url = url_path_join(web_app.settings["base_url"], NAMESPACE) handlers = list() # File templates list_templates = config.file_templates ## Create the loaders templates = dict() for template in list_templates: name = template.name if name in templates: logger.warning( f"Template '{name}' already exists; it will be ignored.") continue else: new_template = { "loader": None, "files": template.files, } location = Path(template.location) if location.exists() and location.is_dir(): new_template["loader"] = FileSystemLoader(str(location)) elif len(template.module) > 0: try: new_template["loader"] = PackageLoader( template.module, package_path=str(location)) except ModuleNotFoundError: logger.warning( f"Unable to find module '{template.module}'") if new_template["loader"] is None: logger.warning(f"Unable to load templates '{name}'.") continue templates[name] = new_template env = Environment( loader=PrefixLoader( {name: t["loader"] for name, t in templates.items()}), extensions=jinja2_extensions, ) ## Create the handlers file_settings = list() for name, template in templates.items(): filenames = set() for file in template["files"]: pfile = Path(file.template) suffixes = "".join(pfile.suffixes) short_name = pfile.as_posix()[:-(len(suffixes))] if short_name in filenames: logger.warning( f"Template '{name}/{pfile.as_posix()}' skipped as it has the same name than another template." ) continue filenames.add(short_name) endpoint = quote("/".join((name, short_name)), safe="") handlers.append(( url_path_join( base_url, r"files/{:s}{:s}".format(endpoint, path_regex), ), FileTemplatesHandler, { "default_name": file.default_name, "template": env.get_template(f"{name}/{pfile.as_posix()}"), }, )) destination = (None if file.destination == Path("") else file.destination.as_posix()) file_settings.append({ "name": file.template_name or endpoint, "endpoint": endpoint, "destination": destination, "icon": file.icon, "schema": file.schema if len(file.schema) else None, }) project_template = config.project_template if project_template is None or project_template.template is None: project_settings = None else: handlers.append(( url_path_join(base_url, r"projects{:s}".format(path_regex)), ProjectsHandler, { "template": project_template }, )) default_path = (None if project_template.default_path == Path("") else project_template.default_path.as_posix()) project_settings = { "configurationFilename": project_template.configuration_filename, "defaultCondaPackages": project_template.conda_pkgs, "defaultPath": default_path, "editableInstall": project_template.editable_install, "schema": (project_template.schema if len(project_template.schema) else None), "withGit": True, # TODO make it configurable } handlers.append(( url_path_join(base_url, "settings"), SettingsHandler, { "project_settings": { "fileTemplates": file_settings, "projectTemplate": project_settings, } }, ), ) web_app.add_handlers(host_pattern, handlers)
def initialize(self, *args, **kwargs): """Load configuration settings.""" super().initialize(*args, **kwargs) self.load_config_file(self.config_file) # hook up tornado logging if self.debug: self.log_level = logging.DEBUG tornado.options.options.logging = logging.getLevelName(self.log_level) tornado.log.enable_pretty_logging() self.log = tornado.log.app_log self.init_pycurl() # initialize kubernetes config if self.builder_required: try: kubernetes.config.load_incluster_config() except kubernetes.config.ConfigException: kubernetes.config.load_kube_config() self.tornado_settings[ "kubernetes_client"] = kubernetes.client.CoreV1Api() # times 2 for log + build threads self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2) # default executor for asyncifying blocking calls (e.g. to kubernetes, docker). # this should not be used for long-running requests self.executor = ThreadPoolExecutor(self.executor_threads) jinja_options = dict(autoescape=True, ) template_paths = [self.template_path] base_template_path = self._template_path_default() if base_template_path not in template_paths: # add base templates to the end, so they are looked up at last after custom templates template_paths.append(base_template_path) loader = ChoiceLoader([ # first load base templates with prefix PrefixLoader({'templates': FileSystemLoader([base_template_path])}, '/'), # load all templates FileSystemLoader(template_paths) ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry and self.builder_required: registry = DockerRegistry(self.docker_auth_host, self.docker_token_url, self.docker_registry_host) else: registry = None self.launcher = Launcher( parent=self, hub_url=self.hub_url, hub_api_token=self.hub_api_token, ) self.tornado_settings.update({ "docker_push_secret": self.docker_push_secret, "docker_image_prefix": self.docker_image_prefix, "github_auth_token": self.github_auth_token, "debug": self.debug, 'hub_url': self.hub_url, 'hub_api_token': self.hub_api_token, 'launcher': self.launcher, 'appendix': self.appendix, "build_namespace": self.build_namespace, "builder_image_spec": self.builder_image_spec, 'build_node_selector': self.build_node_selector, 'build_pool': self.build_pool, 'per_repo_quota': self.per_repo_quota, 'repo_providers': self.repo_providers, 'use_registry': self.use_registry, 'registry': registry, 'traitlets_config': self.config, 'google_analytics_code': self.google_analytics_code, 'google_analytics_domain': self.google_analytics_domain, 'jinja2_env': jinja_env, 'build_memory_limit': self.build_memory_limit, 'build_docker_host': self.build_docker_host, 'base_url': self.base_url, "static_path": os.path.join(os.path.dirname(__file__), "static"), 'static_url_prefix': url_path_join(self.base_url, 'static/'), 'template_variables': self.template_variables, 'executor': self.executor, }) handlers = [ (r'/metrics', MetricsHandler), (r"/build/([^/]+)/(.+)", BuildHandler), (r"/v2/([^/]+)/(.+)", ParameterizedMainHandler), (r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler), # for backward-compatible mybinder.org badge URLs # /assets/images/badge.svg (r'/assets/(images/badge\.svg)', tornado.web.StaticFileHandler, { 'path': self.tornado_settings['static_path'] }), # /badge.svg (r'/(badge\.svg)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), # /favicon_XXX.ico (r'/(favicon\_fail\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/(favicon\_success\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/(favicon\_building\.ico)', tornado.web.StaticFileHandler, { 'path': os.path.join(self.tornado_settings['static_path'], 'images') }), (r'/', MainHandler), (r'.*', Custom404), ] if self.extra_static_path: handlers.insert(-1, (re.escape(self.extra_static_url_prefix) + r"(.*)", tornado.web.StaticFileHandler, { 'path': self.extra_static_path })) handlers = self.add_url_prefix(self.base_url, handlers) self.tornado_app = tornado.web.Application(handlers, **self.tornado_settings)
from flask import Flask from flask_compress import Compress from flask_talisman import Talisman from jinja2 import ChoiceLoader, PackageLoader, PrefixLoader app = Flask(__name__, static_url_path="/assets") app.jinja_loader = ChoiceLoader([ PackageLoader("app"), PrefixLoader( {"govuk_frontend_jinja": PackageLoader("govuk_frontend_jinja")}), ]) app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True csp = { "default-src": "'self'", "script-src": [ "'self'", "'sha256-+6WnXIl4mbFTCARd8N3COQmT3bJJmo32N8q8ZSQAIcU='", "'sha256-l1eTVSK8DTnK8+yloud7wZUqFrI0atVo6VlC6PJvYaQ='", ], "img-src": ["data:", "'self'"], } Compress(app) Talisman(app, content_security_policy=csp) from app import routes
if PREFIX: PREFIX = PREFIX.rstrip("/") if PREFIX and not PREFIX.startswith("/"): PREFIX = "/" + PREFIX DEBUG = config.get("general", "debug_mode") or "-d" in sys.argv or "--debug" in sys.argv bottle.debug(DEBUG) cache = join("tmp", "jinja_cache") if not exists(cache): makedirs(cache) bcc = FileSystemBytecodeCache(cache, '%s.cache') loader = PrefixLoader({ "default": FileSystemLoader(join(PROJECT_DIR, "templates", "default")), "mobile": FileSystemLoader(join(PROJECT_DIR, "templates", "mobile")), 'js': FileSystemLoader(join(PROJECT_DIR, 'media', 'js')) }) env = Environment(loader=loader, extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape'], trim_blocks=True, auto_reload=True, bytecode_cache=bcc) # Filter env.filters["type"] = lambda x: str(type(x)) env.filters["formatsize"] = format_size env.filters["getitem"] = lambda x, y: x.__getitem__(y) if not PREFIX: env.filters["url"] = lambda x: x else: env.filters["url"] = lambda x: PREFIX + x if x.startswith("/") else x
def test_correct_prefix_loader_name(self, env): env = Environment(loader=PrefixLoader({'foo': DictLoader({})})) with pytest.raises(TemplateNotFound) as e: env.get_template('foo/bar.html') assert e.value.name == 'foo/bar.html'
import os from flask import Flask, render_template_string, request from jinja2 import FileSystemLoader, PrefixLoader app = Flask(__name__) app.jinja_loader = PrefixLoader({ "govuk_frontend_jinja": FileSystemLoader(searchpath=os.path.join( os.path.dirname(__file__), "../../govuk_frontend_jinja/templates")) }) # Template route @app.route("/template", methods=["POST"]) def template(): data = request.json # Construct a page template which can override any of the blocks if they are specified # This doesn't need to be inline - it could be it's own file template = """ {% extends "govuk_frontend_jinja/template.html" %} {% block pageTitle %}{% if pageTitle %}{{ pageTitle }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block headIcons %}{% if headIcons %}{{ headIcons }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block head %}{% if head %}{{ head }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block bodyStart %}{% if bodyStart %}{{ bodyStart }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block skipLink %}{% if skipLink %}{{ skipLink }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block header %}{% if header %}{{ header }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block main %}{% if main %}{{ main }}{% else %}{{ super() }}{% endif %}{% endblock %} {% block beforeContent %}{% if beforeContent %}{{ beforeContent }}{% else %}{{ super() }}{% endif %}{% endblock %} # noqa: E501
def init_app(self, app, loader=PrefixLoader({}), pkg_name=None, pkg_path=None): super(Jinja, self).init_app(app, loader, pkg_name, pkg_path) self.add_env("host", app.config.get("HOST", "")) self.add_env("static_url", app.config.get("STATIC_URL", ""))
def app(self): app = Sanic(__name__) default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) template_paths.extend([ plugin["templates_path"] for plugin in get_plugins(pm) if plugin["templates_path"] ]) template_paths.append(default_templates) template_loader = ChoiceLoader([ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader({"default": FileSystemLoader(default_templates)}, delimiter=":"), ]) self.jinja_env = Environment(loader=template_loader, autoescape=True) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters[ "quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class pm.hook.prepare_jinja2_environment(env=self.jinja_env) app.add_route(IndexView.as_view(self), "/<as_format:(\.jsono?)?$>") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires app.add_route(favicon, "/favicon.ico") app.static("/-/static/", str(app_root / "datasette" / "static")) for path, dirname in self.static_mounts: app.static(path, dirname) # Mount any plugin static/ directories for plugin in get_plugins(pm): if plugin["static_path"]: modpath = "/-/static-plugins/{}/".format(plugin["name"]) app.static(modpath, plugin["static_path"]) app.add_route( JsonDataView.as_view(self, "inspect.json", self.inspect), "/-/inspect<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "metadata.json", lambda: self.metadata), "/-/metadata<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "versions.json", self.versions), "/-/versions<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "plugins.json", self.plugins), "/-/plugins<as_format:(\.json)?$>", ) app.add_route( JsonDataView.as_view(self, "config.json", lambda: self.config), "/-/config<as_format:(\.json)?$>", ) app.add_route(DatabaseView.as_view(self), "/<db_name:[^/\.]+?><as_format:(\.jsono?|\.csv)?$>") app.add_route(DatabaseDownload.as_view(self), "/<db_name:[^/]+?><as_db:(\.db)$>") app.add_route( TableView.as_view(self), "/<db_name:[^/]+>/<table_and_format:[^/]+?$>", ) app.add_route( RowView.as_view(self), "/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(\.jsono?)?$>", ) self.register_custom_units() @app.exception(Exception) def on_exception(request, exception): title = None help = None if isinstance(exception, NotFound): status = 404 info = {} message = exception.args[0] elif isinstance(exception, InvalidUsage): status = 405 info = {} message = exception.args[0] elif isinstance(exception, DatasetteError): status = exception.status info = exception.error_dict message = exception.message if exception.messagge_is_html: message = Markup(message) title = exception.title else: status = 500 info = {} message = str(exception) traceback.print_exc() templates = ["500.html"] if status != 500: templates = ["{}.html".format(status)] + templates info.update({ "ok": False, "error": message, "status": status, "title": title }) if request.path.split("?")[0].endswith(".json"): return response.json(info, status=status) else: template = self.jinja_env.select_template(templates) return response.html(template.render(info), status=status) return app
from conveyancer_ui.landregistry_flask import LandRegistryFlask from jinja2 import PackageLoader from jinja2 import PrefixLoader app = LandRegistryFlask(__name__, template_folder='templates', static_folder='assets/dist', static_url_path='/ui' ) # Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example) app.jinja_loader = PrefixLoader({ 'app': PackageLoader('conveyancer_ui') }) app.config.from_pyfile("config.py") @app.context_processor def inject_global_values(): """Inject global template values Use this to inject values into the templates that are used globally. This might be things such as google analytics keys, or even the current username """ return dict( service_name='Digital Street UI' )
try: filters.update(app.filters.filters) except AttributeError: pass class RelEnvironment(Environment): """Override join_path() to enable relative template paths.""" def join_path(self, template, parent): if template.startswith('/'): return template return os.path.normpath( os.path.join('/', os.path.dirname(parent), template)) jinja_env = RelEnvironment(loader=PrefixLoader(_loaders), autoescape=True, cache_size=-1, extensions=['jinja2.ext.loopcontrols']) for name, fn in filters.iteritems(): jinja_env.filters[name] = fn def render(names, **context): """ Render a string from template. Usage: string = render_string('template.html', var1='value 1', var2='value 2') """ if isinstance(names, (str, unicode)): names = [names]
def __init__(self, context, settings, path, theme, output_path, readers_cache_name='',quiet=False, **kwargs): self.settings = settings self.logger = logging.getLogger(__name__) self.stats = defaultdict(int) self.init_pool(ncpu=1) self.context = context self.path = path self.theme = theme self.output_path = output_path self.readers_cache_name = readers_cache_name self._templates = {} self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES']) # self.logger.setLevel(logging.INFO) self.logger.setLevel(logging.DEBUG) theme_templates_path = os.path.expanduser( os.path.join(self.theme, 'templates')) self._templates_path.append(theme_templates_path) theme_loader = FileSystemLoader(theme_templates_path) simple_theme_path = os.path.dirname(os.path.abspath(__file__)) simple_loader = FileSystemLoader( os.path.join(simple_theme_path, "themes", "simple", "templates")) self.env = Environment( loader=ChoiceLoader([ FileSystemLoader(self._templates_path), simple_loader, # implicit inheritance PrefixLoader({ '!simple': simple_loader, '!theme': theme_loader }) # explicit ones ]), **self.settings['JINJA_ENVIRONMENT'] ) # 修正输出位置 print(self.settings['OUTPUT_PATH'], self.settings.get('GALLERY_DEST')) dst_path = os.path.join(self.settings['OUTPUT_PATH'], self.settings['GALLERY_DEST']) settings['GALLERY_DEST'] = dst_path self.settings['GALLERY_DEST'] = dst_path check_or_create_dir(settings['GALLERY_DEST']) # Build the list of directories with images albums = self.albums = {} src_path = self.settings['GALLERY_SOURCE'] ignore_dirs = settings['GALLERY_IGNORE_DIRS'] ignore_files = settings['GALLERY_IGNORE_FILES'] progressChars = cycle(["/", "-", "\\", "|"]) show_progress = (not quiet and self.logger.getEffectiveLevel() >= logging.WARNING and os.isatty(sys.stdout.fileno())) self.progressbar_target = None if show_progress else Devnull() print(f'looking for {os.path.abspath(src_path)}') print(src_path) for path, dirs, files in os.walk(src_path, followlinks=True, topdown=False): print(path) if show_progress: print("\rCollecting albums " + next(progressChars), end="") relpath = os.path.relpath(path, src_path) # print(relpath) # Test if the directory match the ignore_dirs settings if ignore_dirs and any(fnmatch.fnmatch(relpath, ignore) for ignore in ignore_dirs): self.logger.info('Ignoring %s', relpath) continue # Remove files that match the ignore_files settings if ignore_files: files_path = {join(relpath, f) for f in files} for ignore in ignore_files: files_path -= set(fnmatch.filter(files_path, ignore)) self.logger.debug('Files before filtering: %r', files) files = [os.path.split(f)[1] for f in files_path] self.logger.debug('Files after filtering: %r', files) # Remove sub-directories that have been ignored in a previous # iteration (as topdown=False, sub-directories are processed before # their parent for d in dirs[:]: path = join(relpath, d) if relpath != '.' else d if path not in albums.keys(): dirs.remove(d) # 生成album print('Album') print(f'make album {relpath}, {dirs}, {files}') album = Album(relpath, settings, dirs, files, self) if not album.medias and not album.albums: self.logger.info('Skip empty album: %r', album) else: album.create_output_directories() albums[relpath] = album if show_progress: print("\rCollecting albums, done.") # album 排序 with progressbar(albums.values(), label="%16s" % "Sorting albums", file=self.progressbar_target) as progress_albums: for album in progress_albums: album.sort_subdirs(settings['ALBUMS_SORT_ATTR']) with progressbar(albums.values(), label="%16s" % "Sorting media", file=self.progressbar_target) as progress_albums: for album in progress_albums: album.sort_medias(settings['MEDIAS_SORT_ATTR']) self.logger.debug('Albums:\n%r', albums.values())
def __init__( self, files, immutables=None, cache_headers=True, cors=False, inspect_data=None, metadata=None, sqlite_extensions=None, template_dir=None, plugins_dir=None, static_mounts=None, memory=False, config=None, secret=None, version_note=None, config_dir=None, pdb=False, ): assert config_dir is None or isinstance( config_dir, Path), "config_dir= should be a pathlib.Path" self.pdb = pdb self._secret = secret or secrets.token_hex(32) self.files = tuple(files) + tuple(immutables or []) if config_dir: self.files += tuple([str(p) for p in config_dir.glob("*.db")]) if (config_dir and (config_dir / "inspect-data.json").exists() and not inspect_data): inspect_data = json.load((config_dir / "inspect-data.json").open()) if immutables is None: immutable_filenames = [ i["file"] for i in inspect_data.values() ] immutables = [ f for f in self.files if Path(f).name in immutable_filenames ] self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() if memory or not self.files: self.add_database(Database(self, is_memory=True), name=":memory:") # memory_name is a random string so that each Datasette instance gets its own # unique in-memory named database - otherwise unit tests can fail with weird # errors when different instances accidentally share an in-memory database self.add_database(Database(self, memory_name=secrets.token_hex()), name="_internal") self.internal_db_created = False for file in self.files: self.add_database( Database(self, file, is_mutable=file not in self.immutables)) self.cache_headers = cache_headers self.cors = cors metadata_files = [] if config_dir: metadata_files = [ config_dir / filename for filename in ("metadata.json", "metadata.yaml", "metadata.yml") if (config_dir / filename).exists() ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) self._metadata = metadata or {} self.sqlite_functions = [] self.sqlite_extensions = [] for extension in sqlite_extensions or []: # Resolve spatialite, if requested if extension == "spatialite": # Could raise SpatialiteNotFound self.sqlite_extensions.append(find_spatialite()) else: self.sqlite_extensions.append(extension) if config_dir and (config_dir / "templates").is_dir() and not template_dir: template_dir = str((config_dir / "templates").resolve()) self.template_dir = template_dir if config_dir and (config_dir / "plugins").is_dir() and not plugins_dir: plugins_dir = str((config_dir / "plugins").resolve()) self.plugins_dir = plugins_dir if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str( (config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] if config_dir and (config_dir / "config.json").exists(): raise StartupError( "config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: config = json.load((config_dir / "settings.json").open()) self._settings = dict(DEFAULT_SETTINGS, **(config or {})) self.renderers = { } # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( max_workers=self.setting("num_sql_threads")) self.max_returned_rows = self.setting("max_returned_rows") self.sql_time_limit_ms = self.setting("sql_time_limit_ms") self.page_size = self.setting("default_page_size") # Execute plugins in constructor, to ensure they are available # when the rest of `datasette inspect` executes if self.plugins_dir: for filepath in glob.glob(os.path.join(self.plugins_dir, "*.py")): if not os.path.isfile(filepath): continue mod = module_from_path(filepath, name=os.path.basename(filepath)) try: pm.register(mod) except ValueError: # Plugin already registered pass # Configure Jinja default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) plugin_template_paths = [ plugin["templates_path"] for plugin in get_plugins() if plugin["templates_path"] ] template_paths.extend(plugin_template_paths) template_paths.append(default_templates) template_loader = ChoiceLoader([ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader({"default": FileSystemLoader(default_templates)}, delimiter=":"), ]) self.jinja_env = Environment(loader=template_loader, autoescape=True, enable_async=True) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters[ "quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self)
def __init__(self, settings, index_title=''): self.settings = settings self.output_dir = settings['destination'] self.theme = settings['theme'] self.index_title = index_title self.logger = logging.getLogger(__name__) # search the theme in sigal/theme if the given one does not exists if not os.path.exists(self.theme) or \ not os.path.exists(os.path.join(self.theme, 'templates')): self.theme = os.path.join(THEMES_PATH, self.theme) if not os.path.exists(self.theme): raise Exception("Impossible to find the theme %s" % self.theme) self.logger.info("Theme : %s", self.theme) theme_relpath = os.path.join(self.theme, 'templates') default_loader = FileSystemLoader( os.path.join(THEMES_PATH, 'default', 'templates')) # setup jinja env env_options = {'trim_blocks': True, 'autoescape': True} try: if tuple(int(x) for x in jinja2.__version__.split('.')) >= (2, 7): env_options['lstrip_blocks'] = True except ValueError: pass env = Environment( loader=ChoiceLoader([ FileSystemLoader(theme_relpath), default_loader, # implicit inheritance PrefixLoader({'!default': default_loader}) # explicit one ]), **env_options) # handle optional filters.py filters_py = os.path.join(self.theme, 'filters.py') if os.path.exists(filters_py): mod = importlib.import_module(filters_py) for name in dir(mod): if isinstance(getattr(mod, name), types.FunctionType): env.filters[name] = getattr(mod, name) try: self.template = env.get_template(self.template_file) except TemplateNotFound: self.logger.error( 'The template %s was not found in template folder %s.', self.template_file, theme_relpath) sys.exit(1) # Copy the theme files in the output dir self.theme_path = os.path.join(self.output_dir, 'static') if os.path.isdir(self.theme_path): shutil.rmtree(self.theme_path) # FIXME: use dirs_exist_ok when minimum Python is 3.8 shutil.copytree(os.path.join(self.theme, 'static'), self.theme_path) if self.settings['user_css']: if not os.path.exists(self.settings['user_css']): self.logger.error('CSS file %s could not be found', self.settings['user_css']) else: shutil.copy(self.settings['user_css'], self.theme_path)
def __init__(self, context, settings, path, theme, output_path, readers_cache_name='', **kwargs): self.context = context self.settings = settings self.path = path self.theme = theme self.output_path = output_path for arg, value in kwargs.items(): setattr(self, arg, value) self.readers = Readers(self.settings, readers_cache_name) # templates cache self._templates = {} self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES']) theme_templates_path = os.path.expanduser( os.path.join(self.theme, 'templates')) self._templates_path.append(theme_templates_path) theme_loader = FileSystemLoader(theme_templates_path) simple_theme_path = os.path.dirname(os.path.abspath(__file__)) simple_loader = FileSystemLoader( os.path.join(simple_theme_path, "themes", "simple", "templates")) self.env = Environment( loader=ChoiceLoader([ FileSystemLoader(self._templates_path), simple_loader, # implicit inheritance PrefixLoader({ '!simple': simple_loader, '!theme': theme_loader }) # explicit ones ]), **self.settings['JINJA_ENVIRONMENT']) logger.debug('Template list: %s', self.env.list_templates()) # provide utils.strftime as a jinja filter self.env.filters.update({'strftime': DateFormatter()}) # get custom Jinja filters from user settings custom_filters = self.settings['JINJA_FILTERS'] self.env.filters.update(custom_filters) # get custom Jinja globals from user settings custom_globals = self.settings['JINJA_GLOBALS'] self.env.globals.update(custom_globals) # get custom Jinja tests from user settings custom_tests = self.settings['JINJA_TESTS'] self.env.tests.update(custom_tests) signals.generator_init.send(self)
def __init__( self, files, immutables=None, cache_headers=True, cors=False, inspect_data=None, metadata=None, sqlite_extensions=None, template_dir=None, plugins_dir=None, static_mounts=None, memory=False, config=None, secret=None, version_note=None, config_dir=None, ): assert config_dir is None or isinstance( config_dir, Path), "config_dir= should be a pathlib.Path" self._secret = secret or secrets.token_hex(32) self.files = tuple(files) + tuple(immutables or []) if config_dir: self.files += tuple([str(p) for p in config_dir.glob("*.db")]) if (config_dir and (config_dir / "inspect-data.json").exists() and not inspect_data): inspect_data = json.load((config_dir / "inspect-data.json").open()) if immutables is None: immutable_filenames = [ i["file"] for i in inspect_data.values() ] immutables = [ f for f in self.files if Path(f).name in immutable_filenames ] self.inspect_data = inspect_data self.immutables = set(immutables or []) if not self.files: self.files = [MEMORY] elif memory: self.files = (MEMORY, ) + self.files self.databases = collections.OrderedDict() for file in self.files: path = file is_memory = False if file is MEMORY: path = None is_memory = True is_mutable = path not in self.immutables db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) if db.name in self.databases: raise Exception("Multiple files with same stem: {}".format( db.name)) self.add_database(db.name, db) self.cache_headers = cache_headers self.cors = cors metadata_files = [] if config_dir: metadata_files = [ config_dir / filename for filename in ("metadata.json", "metadata.yaml", "metadata.yml") if (config_dir / filename).exists() ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) self._metadata = metadata or {} self.sqlite_functions = [] self.sqlite_extensions = sqlite_extensions or [] if config_dir and (config_dir / "templates").is_dir() and not template_dir: template_dir = str((config_dir / "templates").resolve()) self.template_dir = template_dir if config_dir and (config_dir / "plugins").is_dir() and not plugins_dir: plugins_dir = str((config_dir / "plugins").resolve()) self.plugins_dir = plugins_dir if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str( (config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] if config_dir and (config_dir / "config.json").exists() and not config: config = json.load((config_dir / "config.json").open()) self._config = dict(DEFAULT_CONFIG, **(config or {})) self.renderers = { } # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( max_workers=self.config("num_sql_threads")) self.max_returned_rows = self.config("max_returned_rows") self.sql_time_limit_ms = self.config("sql_time_limit_ms") self.page_size = self.config("default_page_size") # Execute plugins in constructor, to ensure they are available # when the rest of `datasette inspect` executes if self.plugins_dir: for filename in os.listdir(self.plugins_dir): filepath = os.path.join(self.plugins_dir, filename) mod = module_from_path(filepath, name=filename) try: pm.register(mod) except ValueError: # Plugin already registered pass # Configure Jinja default_templates = str(app_root / "datasette" / "templates") template_paths = [] if self.template_dir: template_paths.append(self.template_dir) plugin_template_paths = [ plugin["templates_path"] for plugin in get_plugins() if plugin["templates_path"] ] template_paths.extend(plugin_template_paths) template_paths.append(default_templates) template_loader = ChoiceLoader([ FileSystemLoader(template_paths), # Support {% extends "default:table.html" %}: PrefixLoader({"default": FileSystemLoader(default_templates)}, delimiter=":"), ]) self.jinja_env = Environment(loader=template_loader, autoescape=True, enable_async=True) self.jinja_env.filters["escape_css_string"] = escape_css_string self.jinja_env.filters[ "quote_plus"] = lambda u: urllib.parse.quote_plus(u) self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member pm.hook.prepare_jinja2_environment(env=self.jinja_env) self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32)
def initialize(self, *args, **kwargs): """Load configuration settings.""" super().initialize(*args, **kwargs) self.load_config_file(self.config_file) if self.subapp: return # hook up tornado logging if self.debug: self.log_level = logging.DEBUG tornado.options.options.logging = logging.getLevelName(self.log_level) tornado.log.enable_pretty_logging() self.log = tornado.log.app_log self.init_pycurl() self.init_db() # times 2 for log + build threads self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2) # default executor for asyncifying blocking calls (e.g. to kubernetes, docker). # this should not be used for long-running requests self.executor = ThreadPoolExecutor(self.executor_threads) jinja_options = dict(autoescape=True, ) template_paths = [self.template_path] base_template_path = self._template_path_default() if base_template_path not in template_paths: # add base templates to the end, so they are looked up at last after custom templates template_paths.append(base_template_path) loader = ChoiceLoader([ # first load base templates with prefix PrefixLoader({'templates': FileSystemLoader([base_template_path])}, '/'), # load all templates FileSystemLoader(template_paths) ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry: #registry = DockerRegistry(parent=self) pass else: registry = None self.dashboard = DashboardRepr(parent=self, hub_url=self.hub_url, hub_api_token=self.hub_api_token) #self.event_log = EventLog(parent=self) #for schema_file in glob(os.path.join(HERE, 'event-schemas','*.json')): # with open(schema_file) as f: # self.event_log.register_schema(json.load(f)) self.tornado_settings.update({ "debug": self.debug, 'base_url': self.base_url, "static_path": os.path.join(DATA_FILES_PATH, "static"), 'static_url_prefix': url_path_join(self.base_url, 'static/'), #'template_variables': self.template_variables, #'executor': self.executor, 'build_pool': self.build_pool, 'use_registry': self.use_registry, 'traitlets_config': self.config, 'registry': registry, 'jinja2_env': jinja_env, 'extra_footer_scripts': self.extra_footer_scripts, 'dashboard': self.dashboard, #'event_log': self.event_log, #'normalized_origin': self.normalized_origin }) self.tornado_settings['cookie_secret'] = os.urandom(32) handlers = [ #(r'/metrics', MetricsHandler), #(r'/versions', VersionHandler), #(r"/build/([^/]+)/(.+)", BuildHandler), #(r"/v2/([^/]+)/(.+)", ParameterizedMainHandler), #(r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler), ## for backward-compatible mybinder.org badge URLs ## /assets/images/badge.svg #(r'/assets/(images/badge\.svg)', # tornado.web.StaticFileHandler, # {'path': self.tornado_settings['static_path']}), ## /badge.svg #(r'/(badge\.svg)', # tornado.web.StaticFileHandler, # {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), ## /badge_logo.svg #(r'/(badge\_logo\.svg)', #tornado.web.StaticFileHandler, #{'path': os.path.join(self.tornado_settings['static_path'], 'images')}), # /logo_social.png #(r'/(logo\_social\.png)', #tornado.web.StaticFileHandler, #{'path': os.path.join(self.tornado_settings['static_path'], 'images')}), # /favicon_XXX.ico #(r'/(favicon\_fail\.ico)', # tornado.web.StaticFileHandler, # {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), #(r'/(favicon\_success\.ico)', # tornado.web.StaticFileHandler, # {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), #(r'/(favicon\_building\.ico)', # tornado.web.StaticFileHandler, # {'path': os.path.join(self.tornado_settings['static_path'], 'images')}), #(r'/about', AboutHandler), #(r'/health', HealthHandler, {'hub_url': self.hub_url}), #(self.base_url + r'(?P<user_name>[^/]+)/app/(?P<server_name>[^/]+)?', MainDashboardHandler), #(r'.*', Custom404), ] #handlers = self.add_url_prefix(self.base_url, handlers) #if self.extra_static_path: # handlers.insert(-1, (re.escape(url_path_join(self.base_url, self.extra_static_url_prefix)) + r"(.*)", # tornado.web.StaticFileHandler, # {'path': self.extra_static_path})) oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \ url_path_join(self.base_url, 'oauth_callback') oauth_redirect_uri = urlparse(oauth_redirect_uri).path handlers.insert( -1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler)) self.log.info(self.base_url) self.tornado_app = tornado.web.Application(handlers, **self.tornado_settings)
def test_correct_prefix_loader_name(self): env = Environment(loader=PrefixLoader({'foo': DictLoader({})})) try: env.get_template('foo/bar.html') except TemplateNotFound as e: pass
def __init__(self, loader=None): self.loader = loader self.macro_loaders = [] self.prefix_loader = PrefixLoader({"__macros__": ChoiceLoader(self.macro_loaders)}) super(MacroLoader, self).__init__([self.loader, self.prefix_loader])