def register_loader(self, loader, prefix=None, replace=False):
if not isinstance(self.environment.loader, MacroLoader):
raise Exception("The macro system requires the Jinja loader to be wrapped using an instance of MacroLoader")
if prefix:
loader = PrefixLoader(dict([(prefix, loader)]))
self.environment.loader.macro_loaders.append(loader)
for tpl in loader.list_templates():
self.register_from_template("__macros__/" + tpl, replace)
def initialize(self, *args, **kwargs):
"""Load configuration settings."""
super().initialize(*args, **kwargs)
self.load_config_file(self.config_file)
# hook up tornado logging
if self.debug:
self.log_level = logging.DEBUG
tornado.options.options.logging = logging.getLevelName(self.log_level)
tornado.log.enable_pretty_logging()
self.log = tornado.log.app_log
self.init_pycurl()
# initialize kubernetes config
if self.builder_required:
try:
kubernetes.config.load_incluster_config()
except kubernetes.config.ConfigException:
kubernetes.config.load_kube_config()
self.tornado_settings["kubernetes_client"] = self.kube_client = kubernetes.client.CoreV1Api()
# times 2 for log + build threads
self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2)
# default executor for asyncifying blocking calls (e.g. to kubernetes, docker).
# this should not be used for long-running requests
self.executor = ThreadPoolExecutor(self.executor_threads)
jinja_options = dict(autoescape=True, )
template_paths = [self.template_path]
base_template_path = self._template_path_default()
if base_template_path not in template_paths:
# add base templates to the end, so they are looked up at last after custom templates
template_paths.append(base_template_path)
loader = ChoiceLoader([
# first load base templates with prefix
PrefixLoader({'templates': FileSystemLoader([base_template_path])}, '/'),
# load all templates
FileSystemLoader(template_paths)
])
jinja_env = Environment(loader=loader, **jinja_options)
if self.use_registry and self.builder_required:
registry = DockerRegistry(parent=self)
else:
registry = None
self.launcher = Launcher(
parent=self,
hub_url=self.hub_url,
hub_api_token=self.hub_api_token,
create_user=not self.auth_enabled,
)
self.event_log = EventLog(parent=self)
for schema_file in glob(os.path.join(HERE, 'event-schemas','*.json')):
with open(schema_file) as f:
self.event_log.register_schema(json.load(f))
self.tornado_settings.update({
"push_secret": self.push_secret,
"image_prefix": self.image_prefix,
"debug": self.debug,
'launcher': self.launcher,
'appendix': self.appendix,
"build_namespace": self.build_namespace,
"build_image": self.build_image,
'build_node_selector': self.build_node_selector,
'build_pool': self.build_pool,
'log_tail_lines': self.log_tail_lines,
'per_repo_quota': self.per_repo_quota,
'repo_providers': self.repo_providers,
'use_registry': self.use_registry,
'registry': registry,
'traitlets_config': self.config,
'google_analytics_code': self.google_analytics_code,
'google_analytics_domain': self.google_analytics_domain,
'about_message': self.about_message,
'extra_footer_scripts': self.extra_footer_scripts,
'jinja2_env': jinja_env,
'build_memory_limit': self.build_memory_limit,
'build_docker_host': self.build_docker_host,
'base_url': self.base_url,
"static_path": os.path.join(HERE, "static"),
'static_url_prefix': url_path_join(self.base_url, 'static/'),
'template_variables': self.template_variables,
'executor': self.executor,
'auth_enabled': self.auth_enabled,
'use_named_servers': self.use_named_servers,
'event_log': self.event_log
})
if self.auth_enabled:
self.tornado_settings['cookie_secret'] = os.urandom(32)
handlers = [
(r'/metrics', MetricsHandler),
(r"/build/([^/]+)/(.+)", BuildHandler),
(r"/v2/([^/]+)/(.+)", ParameterizedMainHandler),
(r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler),
# for backward-compatible mybinder.org badge URLs
# /assets/images/badge.svg
(r'/assets/(images/badge\.svg)',
tornado.web.StaticFileHandler,
{'path': self.tornado_settings['static_path']}),
# /badge.svg
(r'/(badge\.svg)',
tornado.web.StaticFileHandler,
{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
# /badge_logo.svg
(r'/(badge\_logo\.svg)',
tornado.web.StaticFileHandler,
{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
# /favicon_XXX.ico
(r'/(favicon\_fail\.ico)',
tornado.web.StaticFileHandler,
{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
(r'/(favicon\_success\.ico)',
tornado.web.StaticFileHandler,
{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
(r'/(favicon\_building\.ico)',
tornado.web.StaticFileHandler,
{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
(r'/about', AboutHandler),
(r'/', MainHandler),
(r'.*', Custom404),
]
handlers = self.add_url_prefix(self.base_url, handlers)
if self.extra_static_path:
handlers.insert(-1, (re.escape(url_path_join(self.base_url, self.extra_static_url_prefix)) + r"(.*)",
tornado.web.StaticFileHandler,
{'path': self.extra_static_path}))
if self.auth_enabled:
oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \
url_path_join(self.base_url, 'oauth_callback')
oauth_redirect_uri = urlparse(oauth_redirect_uri).path
handlers.insert(-1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler))
self.tornado_app = tornado.web.Application(handlers, **self.tornado_settings)
def initialize(self, *args, **kwargs):
"""Load configuration settings."""
super().initialize(*args, **kwargs)
self.load_config_file(self.config_file)
# hook up tornado logging
if self.debug:
self.log_level = logging.DEBUG
tornado.options.options.logging = logging.getLevelName(self.log_level)
tornado.log.enable_pretty_logging()
self.log = tornado.log.app_log
self.init_pycurl()
# initialize kubernetes config
if self.builder_required:
try:
kubernetes.config.load_incluster_config()
except kubernetes.config.ConfigException:
kubernetes.config.load_kube_config()
self.tornado_settings[
"kubernetes_client"] = self.kube_client = kubernetes.client.CoreV1Api(
)
# times 2 for log + build threads
self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2)
# default executor for asyncifying blocking calls (e.g. to kubernetes, docker).
# this should not be used for long-running requests
self.executor = ThreadPoolExecutor(self.executor_threads)
jinja_options = dict(autoescape=True, )
template_paths = [self.template_path]
base_template_path = self._template_path_default()
if base_template_path not in template_paths:
# add base templates to the end, so they are looked up at last after custom templates
template_paths.append(base_template_path)
loader = ChoiceLoader([
# first load base templates with prefix
PrefixLoader({'templates': FileSystemLoader([base_template_path])},
'/'),
# load all templates
FileSystemLoader(template_paths)
])
jinja_env = Environment(loader=loader, **jinja_options)
if self.use_registry:
registry = self.registry_class(parent=self)
else:
registry = None
self.launcher = Launcher(
parent=self,
hub_url=self.hub_url,
hub_url_local=self.hub_url_local,
hub_api_token=self.hub_api_token,
create_user=not self.auth_enabled,
)
self.event_log = EventLog(parent=self)
for schema_file in glob(os.path.join(HERE, 'event-schemas', '*.json')):
with open(schema_file) as f:
self.event_log.register_schema(json.load(f))
self.tornado_settings.update({
"log_function":
log_request,
"push_secret":
self.push_secret,
"image_prefix":
self.image_prefix,
"debug":
self.debug,
"hub_url":
self.hub_url,
"launcher":
self.launcher,
"appendix":
self.appendix,
"ban_networks":
self.ban_networks,
"ban_networks_min_prefix_len":
self.ban_networks_min_prefix_len,
"build_namespace":
self.build_namespace,
"build_image":
self.build_image,
"build_node_selector":
self.build_node_selector,
"build_pool":
self.build_pool,
"build_token_check_origin":
self.build_token_check_origin,
"build_token_secret":
self.build_token_secret,
"build_token_expires_seconds":
self.build_token_expires_seconds,
"sticky_builds":
self.sticky_builds,
"log_tail_lines":
self.log_tail_lines,
"pod_quota":
self.pod_quota,
"per_repo_quota":
self.per_repo_quota,
"per_repo_quota_higher":
self.per_repo_quota_higher,
"repo_providers":
self.repo_providers,
"rate_limiter":
RateLimiter(parent=self),
"use_registry":
self.use_registry,
"build_class":
self.build_class,
"registry":
registry,
"traitlets_config":
self.config,
"google_analytics_code":
self.google_analytics_code,
"google_analytics_domain":
self.google_analytics_domain,
"about_message":
self.about_message,
"banner_message":
self.banner_message,
"extra_footer_scripts":
self.extra_footer_scripts,
"jinja2_env":
jinja_env,
"build_memory_limit":
self.build_memory_limit,
"build_memory_request":
self.build_memory_request,
"build_docker_host":
self.build_docker_host,
"build_docker_config":
self.build_docker_config,
"base_url":
self.base_url,
"badge_base_url":
self.badge_base_url,
"static_path":
os.path.join(HERE, "static"),
"static_url_prefix":
url_path_join(self.base_url, "static/"),
"template_variables":
self.template_variables,
"executor":
self.executor,
"auth_enabled":
self.auth_enabled,
"event_log":
self.event_log,
"normalized_origin":
self.normalized_origin,
"allowed_metrics_ips":
set(map(ipaddress.ip_network, self.allowed_metrics_ips))
})
if self.auth_enabled:
self.tornado_settings['cookie_secret'] = os.urandom(32)
if self.cors_allow_origin:
self.tornado_settings.setdefault(
'headers',
{})['Access-Control-Allow-Origin'] = self.cors_allow_origin
handlers = [
(r'/metrics', MetricsHandler),
(r'/versions', VersionHandler),
(r"/build/([^/]+)/(.+)", BuildHandler),
(r"/v2/([^/]+)/(.+)", ParameterizedMainHandler),
(r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler),
(r'/~([^/]+/.*)', UserRedirectHandler),
# for backward-compatible mybinder.org badge URLs
# /assets/images/badge.svg
(r'/assets/(images/badge\.svg)', tornado.web.StaticFileHandler, {
'path': self.tornado_settings['static_path']
}),
# /badge.svg
(r'/(badge\.svg)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
# /badge_logo.svg
(r'/(badge\_logo\.svg)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
# /logo_social.png
(r'/(logo\_social\.png)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
# /favicon_XXX.ico
(r'/(favicon\_fail\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/(favicon\_success\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/(favicon\_building\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/about', AboutHandler),
(r'/health', HealthHandler, {
'hub_url': self.hub_url_local
}),
(r'/_config', ConfigHandler),
(r'/', MainHandler),
(r'.*', Custom404),
]
handlers = self.add_url_prefix(self.base_url, handlers)
if self.extra_static_path:
handlers.insert(-1, (re.escape(
url_path_join(self.base_url, self.extra_static_url_prefix)) +
r"(.*)", tornado.web.StaticFileHandler, {
'path': self.extra_static_path
}))
if self.auth_enabled:
oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \
url_path_join(self.base_url, 'oauth_callback')
oauth_redirect_uri = urlparse(oauth_redirect_uri).path
handlers.insert(
-1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler))
self.tornado_app = tornado.web.Application(handlers,
**self.tornado_settings)
def app(self):
"Returns an ASGI app function that serves the whole of Datasette"
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader,
autoescape=True,
enable_async=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self.register_renderers()
routes = []
def add_route(view, regex):
routes.append((regex, view))
# Generate a regex snippet to match all registered renderer file extensions
renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
add_route(favicon, "/favicon.ico")
add_route(asgi_static(app_root / "datasette" / "static"),
r"/-/static/(?P<path>.*)$")
for path, dirname in self.static_mounts:
add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$")
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
add_route(
asgi_static(plugin["static_path"]),
"/-/static-plugins/{}/(?P<path>.*)$".format(
plugin["name"]),
)
# Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611
add_route(
asgi_static(plugin["static_path"]),
"/-/static-plugins/{}/(?P<path>.*)$".format(
plugin["name"].replace("-", "_")),
)
add_route(
JsonDataView.as_asgi(self, "metadata.json",
lambda: self._metadata),
r"/-/metadata(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_asgi(self, "versions.json", self.versions),
r"/-/versions(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_asgi(self, "plugins.json", self.plugins),
r"/-/plugins(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_asgi(self, "config.json", lambda: self._config),
r"/-/config(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_asgi(self, "threads.json", self.threads),
r"/-/threads(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_asgi(self, "databases.json",
self.connected_databases),
r"/-/databases(?P<as_format>(\.json)?)$",
)
add_route(DatabaseDownload.as_asgi(self),
r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$")
add_route(
DatabaseView.as_asgi(self),
r"/(?P<db_name>[^/]+?)(?P<as_format>" + renderer_regex +
r"|.jsono|\.csv)?$",
)
add_route(
TableView.as_asgi(self),
r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
)
add_route(
RowView.as_asgi(self),
r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>"
+ renderer_regex + r")?$",
)
self.register_custom_units()
async def setup_db():
# First time server starts up, calculate table counts for immutable databases
for dbname, database in self.databases.items():
if not database.is_mutable:
await database.table_counts(limit=60 * 60 * 1000)
asgi = AsgiLifespan(AsgiTracer(DatasetteRouter(self, routes)),
on_startup=setup_db)
for wrapper in pm.hook.asgi_wrapper(datasette=self):
asgi = wrapper(asgi)
return asgi
import os
from jinja2 import Environment
from jinja2 import PackageLoader
from jinja2 import PrefixLoader
from . import conf
from . import util
__template_packages = {'default': PackageLoader('redreport', 'templates')}
for plugin in conf['plugins']:
try:
module = __import__(plugin)
except ImportError:
pass
else:
if os.path.exists(os.path.join(module.__path__), 'templates'):
__template_packages[plugin] = PackageLoader(plugin, 'templates')
env = Environment(loader=PrefixLoader(__template_packages))
env.filters['dateformat'] = util.dateformat
env.filters['monthformat'] = util.monthformat
from lender_ui.landregistry_flask import LandRegistryFlask
from jinja2 import PackageLoader
from jinja2 import PrefixLoader
app = LandRegistryFlask(
__name__,
# template_folder='templates',
static_folder='assets/dist',
static_url_path='/ui')
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = PrefixLoader({'app': PackageLoader('lender_ui')})
app.config.from_pyfile("config.py")
@app.context_processor
def inject_global_values():
"""Inject global template values
Use this to inject values into the templates that are used globally.
This might be things such as google analytics keys, or even the current username
"""
return dict(service_name='Lender UI')
def app(self):
class TracingSanic(Sanic):
async def handle_request(self, request, write_callback,
stream_callback):
if request.args.get("_trace"):
request["traces"] = []
request["trace_start"] = time.time()
with capture_traces(request["traces"]):
await super().handle_request(request, write_callback,
stream_callback)
else:
await super().handle_request(request, write_callback,
stream_callback)
app = TracingSanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self.register_renderers()
# Generate a regex snippet to match all registered renderer file extensions
renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "metadata.json",
lambda: self._metadata),
r"/-/metadata<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
r"/-/versions<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
r"/-/plugins<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self._config),
r"/-/config<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "databases.json",
self.connected_databases),
r"/-/databases<as_format:(\.json)?$>",
)
app.add_route(DatabaseDownload.as_view(self),
r"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(
DatabaseView.as_view(self),
r"/<db_name:[^/]+?><as_format:(" + renderer_regex +
r"|.jsono|\.csv)?$>",
)
app.add_route(TableView.as_view(self),
r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>")
app.add_route(
RowView.as_view(self),
r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(" +
renderer_regex + r")?$>",
)
self.register_custom_units()
# On 404 with a trailing slash redirect to path without that slash:
# pylint: disable=unused-variable
@app.middleware("response")
def redirect_on_404_with_trailing_slash(request, original_response):
if original_response.status == 404 and request.path.endswith("/"):
path = request.path.rstrip("/")
if request.query_string:
path = "{}?{}".format(path, request.query_string)
return response.redirect(path)
@app.middleware("response")
async def add_traces_to_response(request, response):
if request.get("traces") is None:
return
traces = request["traces"]
trace_info = {
"request_duration_ms":
1000 * (time.time() - request["trace_start"]),
"sum_trace_duration_ms":
sum(t["duration_ms"] for t in traces),
"num_traces":
len(traces),
"traces":
traces,
}
if "text/html" in response.content_type and b"</body>" in response.body:
extra = json.dumps(trace_info, indent=2)
extra_html = "<pre>{}</pre></body>".format(extra).encode(
"utf8")
response.body = response.body.replace(b"</body>", extra_html)
elif "json" in response.content_type and response.body.startswith(
b"{"):
data = json.loads(response.body.decode("utf8"))
if "_trace" not in data:
data["_trace"] = trace_info
response.body = json.dumps(data).encode("utf8")
@app.exception(Exception)
def on_exception(request, exception):
title = None
help = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request is not None and request.path.split("?")[0].endswith(
".json"):
r = response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
r = response.html(template.render(info), status=status)
if self.cors:
r.headers["Access-Control-Allow-Origin"] = "*"
return r
# First time server starts up, calculate table counts for immutable databases
@app.listener("before_server_start")
async def setup_db(app, loop):
for dbname, database in self.databases.items():
if not database.is_mutable:
await database.table_counts(limit=60 * 60 * 1000)
return app
#!/usr/bin/env python3
import argparse
import json
import os
from jinja2 import Environment, FileSystemLoader, PrefixLoader
loader = PrefixLoader({
'govuk_frontend_jinja': FileSystemLoader(searchpath=os.path.join(os.path.dirname(__file__),
'../../govuk_frontend_jinja/templates'))
})
env = Environment(loader=loader, autoescape=True)
parser = argparse.ArgumentParser(description='Render a govuk_frontend_jinja template.')
parser.add_argument('--component', metavar='Component name', nargs=1, help='The name of the component')
parser.add_argument('--params', metavar='Params', nargs=1,
help='JSON representation of the params to pass to the component macro')
args = parser.parse_args()
def hyphenated_to_camel(string):
return ''.join(x.capitalize() for x in string.split('-'))
template = env.from_string('''
{{% from "govuk_frontend_jinja/components/" + component + "/macro.html" import govuk{macro_name} %}}
{{{{ govuk{macro_name}(params) }}}}
'''.format(macro_name=hyphenated_to_camel(args.component[0])))
print(template.render(
from demo.landregistry_flask import LandRegistryFlask
from jinja2 import PackageLoader, FileSystemLoader, PrefixLoader
app = LandRegistryFlask(__name__,
template_folder='templates',
static_folder='assets/dist',
static_url_path='/ui')
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = PrefixLoader({
'app':
PackageLoader('demo'),
'components':
FileSystemLoader('hmlr_design_system/components'),
'wtforms_gov':
PackageLoader('demo.custom_extensions.wtforms_helpers')
})
app.config.from_pyfile("config.py")
@app.context_processor
def inject_global_values():
"""Inject global template values
Use this to inject values into the templates that are used globally.
This might be things such as google analytics keys, or even the current username
"""
return dict(service_name='HMLR Design System')
from title_ui.landregistry_flask import LandRegistryFlask
from jinja2 import PackageLoader
from jinja2 import PrefixLoader
app = LandRegistryFlask(__name__,
template_folder='templates',
static_folder='assets/dist',
static_url_path='/ui')
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = PrefixLoader({'app': PackageLoader('title_ui')})
app.config.from_pyfile("config.py")
@app.context_processor
def inject_global_values():
"""Inject global template values
Use this to inject values into the templates that are used globally.
This might be things such as google analytics keys, or even the current username
"""
return dict(service_name='View your title')
from demo.landregistry_flask import LandRegistryFlask
app = LandRegistryFlask(
__name__,
template_folder="templates",
static_folder="assets/dist",
static_url_path="/ui",
)
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = ChoiceLoader([
PackageLoader("demo"),
PrefixLoader({
"components":
FileSystemLoader("hmlr_design_system/components"),
"govuk_frontend_jinja":
PackageLoader("govuk_frontend_jinja"),
"govuk_frontend_wtf":
PackageLoader("govuk_frontend_wtf"),
}),
])
app.config.from_pyfile("config.py")
def parse_path(demo_path):
demo_path = path.relpath(demo_path,
"hmlr_design_system/components").replace(
"/demos", "").replace(".html", "")
path_parts = demo_path.split("/")
return url_for(
"components.component_demo",
from jinja2 import PrefixLoader
from os import getcwd
from os.path import join
import os
app = LandRegistryFlask(__name__,
template_folder='templates',
static_folder='assets/dist',
static_url_path='/static')
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = PrefixLoader({
'app':
PackageLoader('demo.demo'),
'land_registry_elements':
FileSystemLoader('src/land_registry_elements'),
'govuk_elements_jinja_macros':
PackageLoader('govuk_elements_jinja_macros'),
'incubation_area':
FileSystemLoader('src/incubation_area')
})
app.config.from_pyfile("config.py")
@app.context_processor
def inject_global_values():
"""Inject global template values
Use this to inject values into the templates that are used globally.
This might be things such as google analytics keys, or even the current username
"""
def __init__(
self,
files,
immutables=None,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
version_note=None,
):
immutables = immutables or []
self.files = tuple(files) + tuple(immutables)
self.immutables = set(immutables)
if not self.files:
self.files = [MEMORY]
elif memory:
self.files = (MEMORY,) + self.files
self.databases = collections.OrderedDict()
self.inspect_data = inspect_data
for file in self.files:
path = file
is_memory = False
if file is MEMORY:
path = None
is_memory = True
is_mutable = path not in self.immutables
db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory)
if db.name in self.databases:
raise Exception("Multiple files with same stem: {}".format(db.name))
self.add_database(db.name, db)
self.cache_headers = cache_headers
self.cors = cors
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
self.template_dir = template_dir
self.plugins_dir = plugins_dir
self.static_mounts = static_mounts or []
self._config = dict(DEFAULT_CONFIG, **(config or {}))
self.renderers = {} # File extension -> renderer function
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.config("num_sql_threads")
)
self.max_returned_rows = self.config("max_returned_rows")
self.sql_time_limit_ms = self.config("sql_time_limit_ms")
self.page_size = self.config("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
# Configure Jinja
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
plugin_template_paths = [
plugin["templates_path"]
for plugin in get_plugins()
if plugin["templates_path"]
]
template_paths.extend(plugin_template_paths)
template_paths.append(default_templates)
template_loader = ChoiceLoader(
[
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader(
{"default": FileSystemLoader(default_templates)}, delimiter=":"
),
]
)
self.jinja_env = Environment(
loader=template_loader, autoescape=True, enable_async=True
)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self.register_renderers()
def setup_handlers(web_app: "NotebookWebApplication", config: JupyterProject,
logger: logging.Logger):
host_pattern = ".*$"
base_url = url_path_join(web_app.settings["base_url"], NAMESPACE)
handlers = list()
# File templates
list_templates = config.file_templates
## Create the loaders
templates = dict()
for template in list_templates:
name = template.name
if name in templates:
logger.warning(
f"Template '{name}' already exists; it will be ignored.")
continue
else:
new_template = {
"loader": None,
"files": template.files,
}
location = Path(template.location)
if location.exists() and location.is_dir():
new_template["loader"] = FileSystemLoader(str(location))
elif len(template.module) > 0:
try:
new_template["loader"] = PackageLoader(
template.module, package_path=str(location))
except ModuleNotFoundError:
logger.warning(
f"Unable to find module '{template.module}'")
if new_template["loader"] is None:
logger.warning(f"Unable to load templates '{name}'.")
continue
templates[name] = new_template
env = Environment(
loader=PrefixLoader(
{name: t["loader"]
for name, t in templates.items()}),
extensions=jinja2_extensions,
)
## Create the handlers
file_settings = list()
for name, template in templates.items():
filenames = set()
for file in template["files"]:
pfile = Path(file.template)
suffixes = "".join(pfile.suffixes)
short_name = pfile.as_posix()[:-(len(suffixes))]
if short_name in filenames:
logger.warning(
f"Template '{name}/{pfile.as_posix()}' skipped as it has the same name than another template."
)
continue
filenames.add(short_name)
endpoint = quote("/".join((name, short_name)), safe="")
handlers.append((
url_path_join(
base_url,
r"files/{:s}{:s}".format(endpoint, path_regex),
),
FileTemplatesHandler,
{
"default_name": file.default_name,
"template": env.get_template(f"{name}/{pfile.as_posix()}"),
},
))
destination = (None if file.destination == Path("") else
file.destination.as_posix())
file_settings.append({
"name":
file.template_name or endpoint,
"endpoint":
endpoint,
"destination":
destination,
"icon":
file.icon,
"schema":
file.schema if len(file.schema) else None,
})
project_template = config.project_template
if project_template is None or project_template.template is None:
project_settings = None
else:
handlers.append((
url_path_join(base_url, r"projects{:s}".format(path_regex)),
ProjectsHandler,
{
"template": project_template
},
))
default_path = (None if project_template.default_path == Path("") else
project_template.default_path.as_posix())
project_settings = {
"configurationFilename":
project_template.configuration_filename,
"defaultCondaPackages":
project_template.conda_pkgs,
"defaultPath":
default_path,
"editableInstall":
project_template.editable_install,
"schema": (project_template.schema
if len(project_template.schema) else None),
"withGit":
True, # TODO make it configurable
}
handlers.append((
url_path_join(base_url, "settings"),
SettingsHandler,
{
"project_settings": {
"fileTemplates": file_settings,
"projectTemplate": project_settings,
}
},
), )
web_app.add_handlers(host_pattern, handlers)
def initialize(self, *args, **kwargs):
"""Load configuration settings."""
super().initialize(*args, **kwargs)
self.load_config_file(self.config_file)
# hook up tornado logging
if self.debug:
self.log_level = logging.DEBUG
tornado.options.options.logging = logging.getLevelName(self.log_level)
tornado.log.enable_pretty_logging()
self.log = tornado.log.app_log
self.init_pycurl()
# initialize kubernetes config
if self.builder_required:
try:
kubernetes.config.load_incluster_config()
except kubernetes.config.ConfigException:
kubernetes.config.load_kube_config()
self.tornado_settings[
"kubernetes_client"] = kubernetes.client.CoreV1Api()
# times 2 for log + build threads
self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2)
# default executor for asyncifying blocking calls (e.g. to kubernetes, docker).
# this should not be used for long-running requests
self.executor = ThreadPoolExecutor(self.executor_threads)
jinja_options = dict(autoescape=True, )
template_paths = [self.template_path]
base_template_path = self._template_path_default()
if base_template_path not in template_paths:
# add base templates to the end, so they are looked up at last after custom templates
template_paths.append(base_template_path)
loader = ChoiceLoader([
# first load base templates with prefix
PrefixLoader({'templates': FileSystemLoader([base_template_path])},
'/'),
# load all templates
FileSystemLoader(template_paths)
])
jinja_env = Environment(loader=loader, **jinja_options)
if self.use_registry and self.builder_required:
registry = DockerRegistry(self.docker_auth_host,
self.docker_token_url,
self.docker_registry_host)
else:
registry = None
self.launcher = Launcher(
parent=self,
hub_url=self.hub_url,
hub_api_token=self.hub_api_token,
)
self.tornado_settings.update({
"docker_push_secret":
self.docker_push_secret,
"docker_image_prefix":
self.docker_image_prefix,
"github_auth_token":
self.github_auth_token,
"debug":
self.debug,
'hub_url':
self.hub_url,
'hub_api_token':
self.hub_api_token,
'launcher':
self.launcher,
'appendix':
self.appendix,
"build_namespace":
self.build_namespace,
"builder_image_spec":
self.builder_image_spec,
'build_node_selector':
self.build_node_selector,
'build_pool':
self.build_pool,
'per_repo_quota':
self.per_repo_quota,
'repo_providers':
self.repo_providers,
'use_registry':
self.use_registry,
'registry':
registry,
'traitlets_config':
self.config,
'google_analytics_code':
self.google_analytics_code,
'google_analytics_domain':
self.google_analytics_domain,
'jinja2_env':
jinja_env,
'build_memory_limit':
self.build_memory_limit,
'build_docker_host':
self.build_docker_host,
'base_url':
self.base_url,
"static_path":
os.path.join(os.path.dirname(__file__), "static"),
'static_url_prefix':
url_path_join(self.base_url, 'static/'),
'template_variables':
self.template_variables,
'executor':
self.executor,
})
handlers = [
(r'/metrics', MetricsHandler),
(r"/build/([^/]+)/(.+)", BuildHandler),
(r"/v2/([^/]+)/(.+)", ParameterizedMainHandler),
(r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler),
# for backward-compatible mybinder.org badge URLs
# /assets/images/badge.svg
(r'/assets/(images/badge\.svg)', tornado.web.StaticFileHandler, {
'path': self.tornado_settings['static_path']
}),
# /badge.svg
(r'/(badge\.svg)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
# /favicon_XXX.ico
(r'/(favicon\_fail\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/(favicon\_success\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/(favicon\_building\.ico)', tornado.web.StaticFileHandler, {
'path':
os.path.join(self.tornado_settings['static_path'], 'images')
}),
(r'/', MainHandler),
(r'.*', Custom404),
]
if self.extra_static_path:
handlers.insert(-1,
(re.escape(self.extra_static_url_prefix) + r"(.*)",
tornado.web.StaticFileHandler, {
'path': self.extra_static_path
}))
handlers = self.add_url_prefix(self.base_url, handlers)
self.tornado_app = tornado.web.Application(handlers,
**self.tornado_settings)
from flask import Flask
from flask_compress import Compress
from flask_talisman import Talisman
from jinja2 import ChoiceLoader, PackageLoader, PrefixLoader
app = Flask(__name__, static_url_path="/assets")
app.jinja_loader = ChoiceLoader([
PackageLoader("app"),
PrefixLoader(
{"govuk_frontend_jinja": PackageLoader("govuk_frontend_jinja")}),
])
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
csp = {
"default-src":
"'self'",
"script-src": [
"'self'",
"'sha256-+6WnXIl4mbFTCARd8N3COQmT3bJJmo32N8q8ZSQAIcU='",
"'sha256-l1eTVSK8DTnK8+yloud7wZUqFrI0atVo6VlC6PJvYaQ='",
],
"img-src": ["data:", "'self'"],
}
Compress(app)
Talisman(app, content_security_policy=csp)
from app import routes
if PREFIX:
PREFIX = PREFIX.rstrip("/")
if PREFIX and not PREFIX.startswith("/"):
PREFIX = "/" + PREFIX
DEBUG = config.get("general", "debug_mode") or "-d" in sys.argv or "--debug" in sys.argv
bottle.debug(DEBUG)
cache = join("tmp", "jinja_cache")
if not exists(cache):
makedirs(cache)
bcc = FileSystemBytecodeCache(cache, '%s.cache')
loader = PrefixLoader({
"default": FileSystemLoader(join(PROJECT_DIR, "templates", "default")),
"mobile": FileSystemLoader(join(PROJECT_DIR, "templates", "mobile")),
'js': FileSystemLoader(join(PROJECT_DIR, 'media', 'js'))
})
env = Environment(loader=loader, extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape'], trim_blocks=True, auto_reload=True,
bytecode_cache=bcc)
# Filter
env.filters["type"] = lambda x: str(type(x))
env.filters["formatsize"] = format_size
env.filters["getitem"] = lambda x, y: x.__getitem__(y)
if not PREFIX:
env.filters["url"] = lambda x: x
else:
env.filters["url"] = lambda x: PREFIX + x if x.startswith("/") else x
def test_correct_prefix_loader_name(self, env):
env = Environment(loader=PrefixLoader({'foo': DictLoader({})}))
with pytest.raises(TemplateNotFound) as e:
env.get_template('foo/bar.html')
assert e.value.name == 'foo/bar.html'
import os
from flask import Flask, render_template_string, request
from jinja2 import FileSystemLoader, PrefixLoader
app = Flask(__name__)
app.jinja_loader = PrefixLoader({
"govuk_frontend_jinja":
FileSystemLoader(searchpath=os.path.join(
os.path.dirname(__file__), "../../govuk_frontend_jinja/templates"))
})
# Template route
@app.route("/template", methods=["POST"])
def template():
data = request.json
# Construct a page template which can override any of the blocks if they are specified
# This doesn't need to be inline - it could be it's own file
template = """
{% extends "govuk_frontend_jinja/template.html" %}
{% block pageTitle %}{% if pageTitle %}{{ pageTitle }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block headIcons %}{% if headIcons %}{{ headIcons }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block head %}{% if head %}{{ head }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block bodyStart %}{% if bodyStart %}{{ bodyStart }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block skipLink %}{% if skipLink %}{{ skipLink }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block header %}{% if header %}{{ header }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block main %}{% if main %}{{ main }}{% else %}{{ super() }}{% endif %}{% endblock %}
{% block beforeContent %}{% if beforeContent %}{{ beforeContent }}{% else %}{{ super() }}{% endif %}{% endblock %} # noqa: E501
def init_app(self, app, loader=PrefixLoader({}), pkg_name=None, pkg_path=None):
super(Jinja, self).init_app(app, loader, pkg_name, pkg_path)
self.add_env("host", app.config.get("HOST", ""))
self.add_env("static_url", app.config.get("STATIC_URL", ""))
def app(self):
app = Sanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
app.add_route(IndexView.as_view(self), "/<as_format:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "inspect.json", self.inspect),
"/-/inspect<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "metadata.json", lambda: self.metadata),
"/-/metadata<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
"/-/versions<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
"/-/plugins<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self.config),
"/-/config<as_format:(\.json)?$>",
)
app.add_route(DatabaseView.as_view(self),
"/<db_name:[^/\.]+?><as_format:(\.jsono?|\.csv)?$>")
app.add_route(DatabaseDownload.as_view(self),
"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(
TableView.as_view(self),
"/<db_name:[^/]+>/<table_and_format:[^/]+?$>",
)
app.add_route(
RowView.as_view(self),
"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(\.jsono?)?$>",
)
self.register_custom_units()
@app.exception(Exception)
def on_exception(request, exception):
title = None
help = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request.path.split("?")[0].endswith(".json"):
return response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
return response.html(template.render(info), status=status)
return app
from conveyancer_ui.landregistry_flask import LandRegistryFlask
from jinja2 import PackageLoader
from jinja2 import PrefixLoader
app = LandRegistryFlask(__name__,
template_folder='templates',
static_folder='assets/dist',
static_url_path='/ui'
)
# Set Jinja up to be able to load templates from packages (See gadget-govuk-ui for a full example)
app.jinja_loader = PrefixLoader({
'app': PackageLoader('conveyancer_ui')
})
app.config.from_pyfile("config.py")
@app.context_processor
def inject_global_values():
"""Inject global template values
Use this to inject values into the templates that are used globally.
This might be things such as google analytics keys, or even the current username
"""
return dict(
service_name='Digital Street UI'
)
try:
filters.update(app.filters.filters)
except AttributeError:
pass
class RelEnvironment(Environment):
"""Override join_path() to enable relative template paths."""
def join_path(self, template, parent):
if template.startswith('/'):
return template
return os.path.normpath(
os.path.join('/', os.path.dirname(parent), template))
jinja_env = RelEnvironment(loader=PrefixLoader(_loaders),
autoescape=True, cache_size=-1,
extensions=['jinja2.ext.loopcontrols'])
for name, fn in filters.iteritems():
jinja_env.filters[name] = fn
def render(names, **context):
"""
Render a string from template.
Usage:
string = render_string('template.html', var1='value 1', var2='value 2')
"""
if isinstance(names, (str, unicode)):
names = [names]
def __init__(self, context, settings, path, theme, output_path,
readers_cache_name='',quiet=False, **kwargs):
self.settings = settings
self.logger = logging.getLogger(__name__)
self.stats = defaultdict(int)
self.init_pool(ncpu=1)
self.context = context
self.path = path
self.theme = theme
self.output_path = output_path
self.readers_cache_name = readers_cache_name
self._templates = {}
self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES'])
# self.logger.setLevel(logging.INFO)
self.logger.setLevel(logging.DEBUG)
theme_templates_path = os.path.expanduser(
os.path.join(self.theme, 'templates'))
self._templates_path.append(theme_templates_path)
theme_loader = FileSystemLoader(theme_templates_path)
simple_theme_path = os.path.dirname(os.path.abspath(__file__))
simple_loader = FileSystemLoader(
os.path.join(simple_theme_path, "themes", "simple", "templates"))
self.env = Environment(
loader=ChoiceLoader([
FileSystemLoader(self._templates_path),
simple_loader, # implicit inheritance
PrefixLoader({
'!simple': simple_loader,
'!theme': theme_loader
}) # explicit ones
]),
**self.settings['JINJA_ENVIRONMENT']
)
# 修正输出位置
print(self.settings['OUTPUT_PATH'],
self.settings.get('GALLERY_DEST'))
dst_path = os.path.join(self.settings['OUTPUT_PATH'], self.settings['GALLERY_DEST'])
settings['GALLERY_DEST'] = dst_path
self.settings['GALLERY_DEST'] = dst_path
check_or_create_dir(settings['GALLERY_DEST'])
# Build the list of directories with images
albums = self.albums = {}
src_path = self.settings['GALLERY_SOURCE']
ignore_dirs = settings['GALLERY_IGNORE_DIRS']
ignore_files = settings['GALLERY_IGNORE_FILES']
progressChars = cycle(["/", "-", "\\", "|"])
show_progress = (not quiet and
self.logger.getEffectiveLevel() >= logging.WARNING and
os.isatty(sys.stdout.fileno()))
self.progressbar_target = None if show_progress else Devnull()
print(f'looking for {os.path.abspath(src_path)}')
print(src_path)
for path, dirs, files in os.walk(src_path, followlinks=True,
topdown=False):
print(path)
if show_progress:
print("\rCollecting albums " + next(progressChars), end="")
relpath = os.path.relpath(path, src_path)
# print(relpath)
# Test if the directory match the ignore_dirs settings
if ignore_dirs and any(fnmatch.fnmatch(relpath, ignore)
for ignore in ignore_dirs):
self.logger.info('Ignoring %s', relpath)
continue
# Remove files that match the ignore_files settings
if ignore_files:
files_path = {join(relpath, f) for f in files}
for ignore in ignore_files:
files_path -= set(fnmatch.filter(files_path, ignore))
self.logger.debug('Files before filtering: %r', files)
files = [os.path.split(f)[1] for f in files_path]
self.logger.debug('Files after filtering: %r', files)
# Remove sub-directories that have been ignored in a previous
# iteration (as topdown=False, sub-directories are processed before
# their parent
for d in dirs[:]:
path = join(relpath, d) if relpath != '.' else d
if path not in albums.keys():
dirs.remove(d)
# 生成album
print('Album')
print(f'make album {relpath}, {dirs}, {files}')
album = Album(relpath, settings, dirs, files, self)
if not album.medias and not album.albums:
self.logger.info('Skip empty album: %r', album)
else:
album.create_output_directories()
albums[relpath] = album
if show_progress:
print("\rCollecting albums, done.")
# album 排序
with progressbar(albums.values(), label="%16s" % "Sorting albums",
file=self.progressbar_target) as progress_albums:
for album in progress_albums:
album.sort_subdirs(settings['ALBUMS_SORT_ATTR'])
with progressbar(albums.values(), label="%16s" % "Sorting media",
file=self.progressbar_target) as progress_albums:
for album in progress_albums:
album.sort_medias(settings['MEDIAS_SORT_ATTR'])
self.logger.debug('Albums:\n%r', albums.values())
def __init__(
self,
files,
immutables=None,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
secret=None,
version_note=None,
config_dir=None,
pdb=False,
):
assert config_dir is None or isinstance(
config_dir, Path), "config_dir= should be a pathlib.Path"
self.pdb = pdb
self._secret = secret or secrets.token_hex(32)
self.files = tuple(files) + tuple(immutables or [])
if config_dir:
self.files += tuple([str(p) for p in config_dir.glob("*.db")])
if (config_dir and (config_dir / "inspect-data.json").exists()
and not inspect_data):
inspect_data = json.load((config_dir / "inspect-data.json").open())
if immutables is None:
immutable_filenames = [
i["file"] for i in inspect_data.values()
]
immutables = [
f for f in self.files
if Path(f).name in immutable_filenames
]
self.inspect_data = inspect_data
self.immutables = set(immutables or [])
self.databases = collections.OrderedDict()
if memory or not self.files:
self.add_database(Database(self, is_memory=True), name=":memory:")
# memory_name is a random string so that each Datasette instance gets its own
# unique in-memory named database - otherwise unit tests can fail with weird
# errors when different instances accidentally share an in-memory database
self.add_database(Database(self, memory_name=secrets.token_hex()),
name="_internal")
self.internal_db_created = False
for file in self.files:
self.add_database(
Database(self, file, is_mutable=file not in self.immutables))
self.cache_headers = cache_headers
self.cors = cors
metadata_files = []
if config_dir:
metadata_files = [
config_dir / filename
for filename in ("metadata.json", "metadata.yaml",
"metadata.yml")
if (config_dir / filename).exists()
]
if config_dir and metadata_files and not metadata:
with metadata_files[0].open() as fp:
metadata = parse_metadata(fp.read())
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = []
for extension in sqlite_extensions or []:
# Resolve spatialite, if requested
if extension == "spatialite":
# Could raise SpatialiteNotFound
self.sqlite_extensions.append(find_spatialite())
else:
self.sqlite_extensions.append(extension)
if config_dir and (config_dir /
"templates").is_dir() and not template_dir:
template_dir = str((config_dir / "templates").resolve())
self.template_dir = template_dir
if config_dir and (config_dir /
"plugins").is_dir() and not plugins_dir:
plugins_dir = str((config_dir / "plugins").resolve())
self.plugins_dir = plugins_dir
if config_dir and (config_dir /
"static").is_dir() and not static_mounts:
static_mounts = [("static", str(
(config_dir / "static").resolve()))]
self.static_mounts = static_mounts or []
if config_dir and (config_dir / "config.json").exists():
raise StartupError(
"config.json should be renamed to settings.json")
if config_dir and (config_dir /
"settings.json").exists() and not config:
config = json.load((config_dir / "settings.json").open())
self._settings = dict(DEFAULT_SETTINGS, **(config or {}))
self.renderers = {
} # File extension -> (renderer, can_render) functions
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.setting("num_sql_threads"))
self.max_returned_rows = self.setting("max_returned_rows")
self.sql_time_limit_ms = self.setting("sql_time_limit_ms")
self.page_size = self.setting("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filepath in glob.glob(os.path.join(self.plugins_dir, "*.py")):
if not os.path.isfile(filepath):
continue
mod = module_from_path(filepath,
name=os.path.basename(filepath))
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
# Configure Jinja
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
plugin_template_paths = [
plugin["templates_path"] for plugin in get_plugins()
if plugin["templates_path"]
]
template_paths.extend(plugin_template_paths)
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader,
autoescape=True,
enable_async=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self._register_renderers()
self._permission_checks = collections.deque(maxlen=200)
self._root_token = secrets.token_hex(32)
self.client = DatasetteClient(self)
def __init__(self, settings, index_title=''):
self.settings = settings
self.output_dir = settings['destination']
self.theme = settings['theme']
self.index_title = index_title
self.logger = logging.getLogger(__name__)
# search the theme in sigal/theme if the given one does not exists
if not os.path.exists(self.theme) or \
not os.path.exists(os.path.join(self.theme, 'templates')):
self.theme = os.path.join(THEMES_PATH, self.theme)
if not os.path.exists(self.theme):
raise Exception("Impossible to find the theme %s" % self.theme)
self.logger.info("Theme : %s", self.theme)
theme_relpath = os.path.join(self.theme, 'templates')
default_loader = FileSystemLoader(
os.path.join(THEMES_PATH, 'default', 'templates'))
# setup jinja env
env_options = {'trim_blocks': True, 'autoescape': True}
try:
if tuple(int(x) for x in jinja2.__version__.split('.')) >= (2, 7):
env_options['lstrip_blocks'] = True
except ValueError:
pass
env = Environment(
loader=ChoiceLoader([
FileSystemLoader(theme_relpath),
default_loader, # implicit inheritance
PrefixLoader({'!default': default_loader}) # explicit one
]),
**env_options)
# handle optional filters.py
filters_py = os.path.join(self.theme, 'filters.py')
if os.path.exists(filters_py):
mod = importlib.import_module(filters_py)
for name in dir(mod):
if isinstance(getattr(mod, name), types.FunctionType):
env.filters[name] = getattr(mod, name)
try:
self.template = env.get_template(self.template_file)
except TemplateNotFound:
self.logger.error(
'The template %s was not found in template folder %s.',
self.template_file, theme_relpath)
sys.exit(1)
# Copy the theme files in the output dir
self.theme_path = os.path.join(self.output_dir, 'static')
if os.path.isdir(self.theme_path):
shutil.rmtree(self.theme_path)
# FIXME: use dirs_exist_ok when minimum Python is 3.8
shutil.copytree(os.path.join(self.theme, 'static'), self.theme_path)
if self.settings['user_css']:
if not os.path.exists(self.settings['user_css']):
self.logger.error('CSS file %s could not be found',
self.settings['user_css'])
else:
shutil.copy(self.settings['user_css'], self.theme_path)
def __init__(self,
context,
settings,
path,
theme,
output_path,
readers_cache_name='',
**kwargs):
self.context = context
self.settings = settings
self.path = path
self.theme = theme
self.output_path = output_path
for arg, value in kwargs.items():
setattr(self, arg, value)
self.readers = Readers(self.settings, readers_cache_name)
# templates cache
self._templates = {}
self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES'])
theme_templates_path = os.path.expanduser(
os.path.join(self.theme, 'templates'))
self._templates_path.append(theme_templates_path)
theme_loader = FileSystemLoader(theme_templates_path)
simple_theme_path = os.path.dirname(os.path.abspath(__file__))
simple_loader = FileSystemLoader(
os.path.join(simple_theme_path, "themes", "simple", "templates"))
self.env = Environment(
loader=ChoiceLoader([
FileSystemLoader(self._templates_path),
simple_loader, # implicit inheritance
PrefixLoader({
'!simple': simple_loader,
'!theme': theme_loader
}) # explicit ones
]),
**self.settings['JINJA_ENVIRONMENT'])
logger.debug('Template list: %s', self.env.list_templates())
# provide utils.strftime as a jinja filter
self.env.filters.update({'strftime': DateFormatter()})
# get custom Jinja filters from user settings
custom_filters = self.settings['JINJA_FILTERS']
self.env.filters.update(custom_filters)
# get custom Jinja globals from user settings
custom_globals = self.settings['JINJA_GLOBALS']
self.env.globals.update(custom_globals)
# get custom Jinja tests from user settings
custom_tests = self.settings['JINJA_TESTS']
self.env.tests.update(custom_tests)
signals.generator_init.send(self)
def __init__(
self,
files,
immutables=None,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
secret=None,
version_note=None,
config_dir=None,
):
assert config_dir is None or isinstance(
config_dir, Path), "config_dir= should be a pathlib.Path"
self._secret = secret or secrets.token_hex(32)
self.files = tuple(files) + tuple(immutables or [])
if config_dir:
self.files += tuple([str(p) for p in config_dir.glob("*.db")])
if (config_dir and (config_dir / "inspect-data.json").exists()
and not inspect_data):
inspect_data = json.load((config_dir / "inspect-data.json").open())
if immutables is None:
immutable_filenames = [
i["file"] for i in inspect_data.values()
]
immutables = [
f for f in self.files
if Path(f).name in immutable_filenames
]
self.inspect_data = inspect_data
self.immutables = set(immutables or [])
if not self.files:
self.files = [MEMORY]
elif memory:
self.files = (MEMORY, ) + self.files
self.databases = collections.OrderedDict()
for file in self.files:
path = file
is_memory = False
if file is MEMORY:
path = None
is_memory = True
is_mutable = path not in self.immutables
db = Database(self,
path,
is_mutable=is_mutable,
is_memory=is_memory)
if db.name in self.databases:
raise Exception("Multiple files with same stem: {}".format(
db.name))
self.add_database(db.name, db)
self.cache_headers = cache_headers
self.cors = cors
metadata_files = []
if config_dir:
metadata_files = [
config_dir / filename
for filename in ("metadata.json", "metadata.yaml",
"metadata.yml")
if (config_dir / filename).exists()
]
if config_dir and metadata_files and not metadata:
with metadata_files[0].open() as fp:
metadata = parse_metadata(fp.read())
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
if config_dir and (config_dir /
"templates").is_dir() and not template_dir:
template_dir = str((config_dir / "templates").resolve())
self.template_dir = template_dir
if config_dir and (config_dir /
"plugins").is_dir() and not plugins_dir:
plugins_dir = str((config_dir / "plugins").resolve())
self.plugins_dir = plugins_dir
if config_dir and (config_dir /
"static").is_dir() and not static_mounts:
static_mounts = [("static", str(
(config_dir / "static").resolve()))]
self.static_mounts = static_mounts or []
if config_dir and (config_dir / "config.json").exists() and not config:
config = json.load((config_dir / "config.json").open())
self._config = dict(DEFAULT_CONFIG, **(config or {}))
self.renderers = {
} # File extension -> (renderer, can_render) functions
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.config("num_sql_threads"))
self.max_returned_rows = self.config("max_returned_rows")
self.sql_time_limit_ms = self.config("sql_time_limit_ms")
self.page_size = self.config("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
# Configure Jinja
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
plugin_template_paths = [
plugin["templates_path"] for plugin in get_plugins()
if plugin["templates_path"]
]
template_paths.extend(plugin_template_paths)
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader,
autoescape=True,
enable_async=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self._register_renderers()
self._permission_checks = collections.deque(maxlen=200)
self._root_token = secrets.token_hex(32)
def initialize(self, *args, **kwargs):
"""Load configuration settings."""
super().initialize(*args, **kwargs)
self.load_config_file(self.config_file)
if self.subapp:
return
# hook up tornado logging
if self.debug:
self.log_level = logging.DEBUG
tornado.options.options.logging = logging.getLevelName(self.log_level)
tornado.log.enable_pretty_logging()
self.log = tornado.log.app_log
self.init_pycurl()
self.init_db()
# times 2 for log + build threads
self.build_pool = ThreadPoolExecutor(self.concurrent_build_limit * 2)
# default executor for asyncifying blocking calls (e.g. to kubernetes, docker).
# this should not be used for long-running requests
self.executor = ThreadPoolExecutor(self.executor_threads)
jinja_options = dict(autoescape=True, )
template_paths = [self.template_path]
base_template_path = self._template_path_default()
if base_template_path not in template_paths:
# add base templates to the end, so they are looked up at last after custom templates
template_paths.append(base_template_path)
loader = ChoiceLoader([
# first load base templates with prefix
PrefixLoader({'templates': FileSystemLoader([base_template_path])},
'/'),
# load all templates
FileSystemLoader(template_paths)
])
jinja_env = Environment(loader=loader, **jinja_options)
if self.use_registry:
#registry = DockerRegistry(parent=self)
pass
else:
registry = None
self.dashboard = DashboardRepr(parent=self,
hub_url=self.hub_url,
hub_api_token=self.hub_api_token)
#self.event_log = EventLog(parent=self)
#for schema_file in glob(os.path.join(HERE, 'event-schemas','*.json')):
# with open(schema_file) as f:
# self.event_log.register_schema(json.load(f))
self.tornado_settings.update({
"debug":
self.debug,
'base_url':
self.base_url,
"static_path":
os.path.join(DATA_FILES_PATH, "static"),
'static_url_prefix':
url_path_join(self.base_url, 'static/'),
#'template_variables': self.template_variables,
#'executor': self.executor,
'build_pool':
self.build_pool,
'use_registry':
self.use_registry,
'traitlets_config':
self.config,
'registry':
registry,
'jinja2_env':
jinja_env,
'extra_footer_scripts':
self.extra_footer_scripts,
'dashboard':
self.dashboard,
#'event_log': self.event_log,
#'normalized_origin': self.normalized_origin
})
self.tornado_settings['cookie_secret'] = os.urandom(32)
handlers = [
#(r'/metrics', MetricsHandler),
#(r'/versions', VersionHandler),
#(r"/build/([^/]+)/(.+)", BuildHandler),
#(r"/v2/([^/]+)/(.+)", ParameterizedMainHandler),
#(r"/repo/([^/]+)/([^/]+)(/.*)?", LegacyRedirectHandler),
## for backward-compatible mybinder.org badge URLs
## /assets/images/badge.svg
#(r'/assets/(images/badge\.svg)',
# tornado.web.StaticFileHandler,
# {'path': self.tornado_settings['static_path']}),
## /badge.svg
#(r'/(badge\.svg)',
# tornado.web.StaticFileHandler,
# {'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
## /badge_logo.svg
#(r'/(badge\_logo\.svg)',
#tornado.web.StaticFileHandler,
#{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
# /logo_social.png
#(r'/(logo\_social\.png)',
#tornado.web.StaticFileHandler,
#{'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
# /favicon_XXX.ico
#(r'/(favicon\_fail\.ico)',
# tornado.web.StaticFileHandler,
# {'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
#(r'/(favicon\_success\.ico)',
# tornado.web.StaticFileHandler,
# {'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
#(r'/(favicon\_building\.ico)',
# tornado.web.StaticFileHandler,
# {'path': os.path.join(self.tornado_settings['static_path'], 'images')}),
#(r'/about', AboutHandler),
#(r'/health', HealthHandler, {'hub_url': self.hub_url}),
#(self.base_url + r'(?P<user_name>[^/]+)/app/(?P<server_name>[^/]+)?', MainDashboardHandler),
#(r'.*', Custom404),
]
#handlers = self.add_url_prefix(self.base_url, handlers)
#if self.extra_static_path:
# handlers.insert(-1, (re.escape(url_path_join(self.base_url, self.extra_static_url_prefix)) + r"(.*)",
# tornado.web.StaticFileHandler,
# {'path': self.extra_static_path}))
oauth_redirect_uri = os.getenv('JUPYTERHUB_OAUTH_CALLBACK_URL') or \
url_path_join(self.base_url, 'oauth_callback')
oauth_redirect_uri = urlparse(oauth_redirect_uri).path
handlers.insert(
-1, (re.escape(oauth_redirect_uri), HubOAuthCallbackHandler))
self.log.info(self.base_url)
self.tornado_app = tornado.web.Application(handlers,
**self.tornado_settings)
def test_correct_prefix_loader_name(self):
env = Environment(loader=PrefixLoader({'foo': DictLoader({})}))
try:
env.get_template('foo/bar.html')
except TemplateNotFound as e:
pass
def __init__(self, loader=None):
self.loader = loader
self.macro_loaders = []
self.prefix_loader = PrefixLoader({"__macros__": ChoiceLoader(self.macro_loaders)})
super(MacroLoader, self).__init__([self.loader, self.prefix_loader])