def _executeAction(controllerName, action, session):
method = session[Constants.HTTP_METHOD];
if method == Constants.HTTP_METHOD_GET:
params = read_request_vars(fp=session[Constants.GET_REQUEST_CONTENT], environ=session, keep_blank_values=True);
elif method == Constants.HTTP_METHOD_POST:
params = read_request_vars(fp=session[Constants.POST_REQUEST_CONTENT], environ=session, keep_blank_values=True);
else:
raise BadRequestException('Wrong method provided!');
try:
controllerModule = imp.load_source(controllerName, Constants.CONTROLLERS_FOLDER + controllerName + ".py");
controller = getattr(controllerModule, controllerName)(params);
getattr(controller, action)();
loader = FileSystemLoader(Constants.VIEWS_FOLDER + "/" + controllerName);
env = Environment(loader=loader);
template = env.select_template([action + '.html', action + '.xml', action + '.json']);
page = template.render(controller.__dict__);
headers = [('Content-type', guess_type(template.name)[0])];
except TemplateNotFound:
raise BadRequestException('Unable to find a view');
except IOError:
raise BadRequestException('Unable to find the controller');
return Constants.STATUS_OK, headers, page.encode('utf-8');
def build_pages(base_directory):
print "Building pages in %s" % base_directory
root_files = glob(os.path.join(base_directory, '*.markdown'))
pages_built = 0
if os.path.isdir(os.path.join(base_directory, TEMPLATES_DIRECTORY)):
print "Found local template directory"
template_env = Environment(loader=FileSystemLoader(os.path.join(base_directory, TEMPLATES_DIRECTORY)))
else:
print "No Template directory found, using default"
template_env = Environment(loader=PackageLoader('gh-build', 'templates'))
for filename in root_files:
try:
template = template_env.get_template(ROOT_TEMPLATE_NAME)
except TemplateNotFound:
print "root.html template missing in local directory, falling back to default"
template_env = Environment(loader=PackageLoader('gh-build', 'templates'))
template = template_env.get_template(ROOT_TEMPLATE_NAME)
build_page(filename, base_directory, template)
pages_built += 1
pages_files = glob(os.path.join(base_directory, 'pages', '*.markdown'))
for filename in pages_files:
build_page(filename, base_directory, template_env.select_template([PAGE_TEMPLATE_NAME, ROOT_TEMPLATE_NAME]))
pages_built += 1
print "%s pages built" % pages_built
def _executeAction(controllerName, action, session):
method = session[Constants.HTTP_METHOD]
if method == Constants.HTTP_METHOD_GET:
params = read_request_vars(fp=session[Constants.GET_REQUEST_CONTENT],
environ=session,
keep_blank_values=True)
elif method == Constants.HTTP_METHOD_POST:
params = read_request_vars(fp=session[Constants.POST_REQUEST_CONTENT],
environ=session,
keep_blank_values=True)
else:
raise BadRequestException('Wrong method provided!')
try:
controllerModule = imp.load_source(
controllerName,
Constants.CONTROLLERS_FOLDER + controllerName + ".py")
controller = getattr(controllerModule, controllerName)(params)
getattr(controller, action)()
loader = FileSystemLoader(Constants.VIEWS_FOLDER + "/" +
controllerName)
env = Environment(loader=loader)
template = env.select_template(
[action + '.html', action + '.xml', action + '.json'])
page = template.render(controller.__dict__)
headers = [('Content-type', guess_type(template.name)[0])]
except TemplateNotFound:
raise BadRequestException('Unable to find a view')
except IOError:
raise BadRequestException('Unable to find the controller')
return Constants.STATUS_OK, headers, page.encode('utf-8')
def render_j2(data, output_file='index.md'):
env = Environment(loader=PackageLoader(__name__, '.'),
autoescape=select_autoescape([]))
env.filters['linkify'] = linkify
templ = env.select_template(['index.j2'])
return templ.render(data=data.items(),
timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def wrapped_f(*args, **kwargs):
params = f(*args, **kwargs)
jinja = Environment(
loader=FileSystemLoader('webserver/templates'),
extensions=['jinja2.ext.with_']
)
jinja.filters['humanize'] = arrow_humanize
if not filename:
raise cherrypy.HTTPError(500, "Template not defined")
else:
filenames = [filename + '.tlp'] # ..scope royals vs local bastards
if '__template' in params:
if isinstance(params['__template'], basestring):
filenames.insert(0, params['__template'] + '.tlp')
else:
# Don't use +=, we are leaving it *after* __template for precedence
filenames = [s + '.tlp' for s in params['__template']] + filenames
try:
t = jinja.select_template(filenames)
except TemplateNotFound:
raise cherrypy.HTTPError(404, "Template not found")
t.globals['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
t.globals['session'] = cherrypy.session
return t.render(**params)
class Renderer:
"""Class for jinja2 template lookup and rendering."""
def __init__(self, settings):
self.env = Environment()
self.env.filters.update(filters)
loader = FileSystemLoader(settings["templates"])
self.env.loader = loader
def render(self, out, templates, args):
"""Select the first available template and render to `out`."""
template = self.lookup(templates)
text = template.render(**args)
os.makedirs(os.path.dirname(out), exist_ok=True)
with open(out, "w") as _file:
_file.write(text)
return text
def lookup(self, templates):
"""Lookup a template from the template directory.
Templates are selected based on `templates` list
order. Available templates extensions are `html` and
`xml. `html` files take template selection precendence over
`xml` files.
"""
exts = [".html", ".xml"]
template_files = []
for f, ext in itertools.product(templates, exts):
if f is not None:
template_files.append(str(f) + ext)
return self.env.select_template(template_files)
@staticmethod
def render_from_string(string, args=None):
"""Render a string template with access to custom filters."""
env = Environment()
env.filters.update(filters)
template = env.from_string(string)
text = template.render(**args) if args else template.render()
return text
class TemplateHelper(object):
def __init__(self, loader):
self._env = Environment(loader=loader)
def get_template(self, name):
filename = '{}.jinja'.format(name)
return _Template(self._env.get_template(filename))
def get_customizable_template(self, name, custom_part):
filename = '{}.jinja'.format(name)
filename_custom = '{}-{}.jinja'.format(name, custom_part)
return _Template(self._env.select_template([filename_custom,
filename]))
def get_legacy_contexts_conf(self):
# TODO return contextsconf as an OrderedRawConf like previously...
pass
def _render_template(distro, loader, **kwargs):
env = Environment(
loader=loader,
trim_blocks=True,
lstrip_blocks=True,
)
env.filters['dedent'] = textwrap.dedent
template_name = 'sysprep-{0}.j2'.format(distro)
template = env.select_template([template_name, 'sysprep-base.j2'])
sysprep_content = template.render(guestfs_ver=_guestfs_version(), **kwargs)
with tempfile.NamedTemporaryFile(mode='w', delete=False) as sysprep_file:
sysprep_file.write('# {0}\n'.format(template.name))
sysprep_file.write(sysprep_content)
LOGGER.debug(('Generated sysprep template '
'at {0}:\n{1}').format(sysprep_file.name, sysprep_content))
return sysprep_file.name
def _render_template(distro, loader, **kwargs):
env = Environment(
loader=loader,
trim_blocks=True,
lstrip_blocks=True,
)
env.filters['dedent'] = textwrap.dedent
template_name = 'sysprep-{0}.j2'.format(distro)
template = env.select_template([template_name, 'sysprep-base.j2'])
sysprep_content = template.render(guestfs_ver=_guestfs_version(), **kwargs)
with tempfile.NamedTemporaryFile(delete=False) as sysprep_file:
sysprep_file.write('# {0}\n'.format(template.name))
sysprep_file.write(sysprep_content)
LOGGER.debug(
('Generated sysprep template '
'at {0}:\n{1}').format(sysprep_file.name, sysprep_content)
)
return sysprep_file.name
class render_jinja:
"""Rendering interface to Jinja2 Templates
Example:
render= render_jinja('templates')
render.hello(name='jinja2')
"""
def __init__(self, *a, **kwargs):
extensions = kwargs.pop('extensions', [])
globals = kwargs.pop('globals', {})
from jinja2 import Environment, FileSystemLoader
self._lookup = Environment(loader=FileSystemLoader(*a, **kwargs),
extensions=extensions)
self._lookup.globals.update(globals)
def __getattr__(self, name):
paths = [name + '.' + ext for ext in ['html', 'xml']]
t = self._lookup.select_template(paths)
return t.render
def render_apache_tomcat_dockerfiles(data,
config,
update_all_versions=False,
force_update=False):
env = Environment(loader=FileSystemLoader(os.path.abspath('templates')))
repository_name = config['repository_name']
base_repositories = config['base_repositories']
template_files = config['templates']
registries = config.get('registries')
common_files = config.get('common_files')
versions = data['versions'].keys()
if update_all_versions:
versions_to_update = versions
else:
versions_to_update = filter_latest_versions(
versions,
version_constraints=config.get('version_constraints'),
normalize_version=normalize_version_to_semver)
base_repository_info_list = get_base_repository_info(config)
for version in versions_to_update:
version_files = data['versions'][version]
for base_repository_info in base_repository_info_list:
base_repository_full_repo = base_repository_info['full_repo']
base_repository_tag_groups = base_repository_info['tag_groups']
base_repository_name = base_repository_info['name']
for base_repository_tag_group in base_repository_tag_groups:
base_repository_main_tag = base_repository_tag_group[0]
base_image_name = base_repository_full_repo + ':' + base_repository_main_tag
dockerfile_context = os.path.join(
os.getcwd(), version,
base_repository_name + base_repository_main_tag)
tags = [version]
for base_repository_tag in base_repository_tag_group:
tags.append(version + '-' + base_repository_name +
base_repository_tag)
version_info = semver.parse_version_info(
normalize_version_to_semver(version))
base_os = re.compile(
'centos|alpine|ubuntu|debian|fedora|rhel').search(
base_repository_name +
base_repository_main_tag).group(0)
render_data = {
'base_image_name': base_image_name,
'base_os': base_os,
'base_repository_name': base_repository_name,
'config': config,
'files': version_files,
'registries': registries,
'repository_name': repository_name,
'tags': tags,
'version': version,
'version_info': version_info,
}
pprint(render_data)
for template_file in template_files:
template_filenames = [
template_file + '.' + base_os + '.j2',
template_file + '.j2'
]
template = env.select_template(template_filenames)
template_render_path = os.path.join(
dockerfile_context, template_file)
if not os.path.exists(
template_render_path) or force_update:
write_file(template_render_path,
template.render(render_data))
print 'Rendered template: ' + template_render_path
for common_file in common_files:
common_file_path = os.path.join(dockerfile_context,
common_file)
if not os.path.exists(common_file_path) or force_update:
shutil.copy2(common_file, common_file_path)
print 'Copied file: ' + common_file_path
class Datasette:
# Message constants:
INFO = 1
WARNING = 2
ERROR = 3
def __init__(
self,
files,
immutables=None,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
secret=None,
version_note=None,
config_dir=None,
):
assert config_dir is None or isinstance(
config_dir, Path), "config_dir= should be a pathlib.Path"
self._secret = secret or secrets.token_hex(32)
self.files = tuple(files) + tuple(immutables or [])
if config_dir:
self.files += tuple([str(p) for p in config_dir.glob("*.db")])
if (config_dir and (config_dir / "inspect-data.json").exists()
and not inspect_data):
inspect_data = json.load((config_dir / "inspect-data.json").open())
if immutables is None:
immutable_filenames = [
i["file"] for i in inspect_data.values()
]
immutables = [
f for f in self.files
if Path(f).name in immutable_filenames
]
self.inspect_data = inspect_data
self.immutables = set(immutables or [])
if not self.files:
self.files = [MEMORY]
elif memory:
self.files = (MEMORY, ) + self.files
self.databases = collections.OrderedDict()
for file in self.files:
path = file
is_memory = False
if file is MEMORY:
path = None
is_memory = True
is_mutable = path not in self.immutables
db = Database(self,
path,
is_mutable=is_mutable,
is_memory=is_memory)
if db.name in self.databases:
raise Exception("Multiple files with same stem: {}".format(
db.name))
self.add_database(db.name, db)
self.cache_headers = cache_headers
self.cors = cors
metadata_files = []
if config_dir:
metadata_files = [
config_dir / filename
for filename in ("metadata.json", "metadata.yaml",
"metadata.yml")
if (config_dir / filename).exists()
]
if config_dir and metadata_files and not metadata:
with metadata_files[0].open() as fp:
metadata = parse_metadata(fp.read())
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
if config_dir and (config_dir /
"templates").is_dir() and not template_dir:
template_dir = str((config_dir / "templates").resolve())
self.template_dir = template_dir
if config_dir and (config_dir /
"plugins").is_dir() and not plugins_dir:
plugins_dir = str((config_dir / "plugins").resolve())
self.plugins_dir = plugins_dir
if config_dir and (config_dir /
"static").is_dir() and not static_mounts:
static_mounts = [("static", str(
(config_dir / "static").resolve()))]
self.static_mounts = static_mounts or []
if config_dir and (config_dir / "config.json").exists() and not config:
config = json.load((config_dir / "config.json").open())
self._config = dict(DEFAULT_CONFIG, **(config or {}))
self.renderers = {
} # File extension -> (renderer, can_render) functions
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.config("num_sql_threads"))
self.max_returned_rows = self.config("max_returned_rows")
self.sql_time_limit_ms = self.config("sql_time_limit_ms")
self.page_size = self.config("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
# Configure Jinja
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
plugin_template_paths = [
plugin["templates_path"] for plugin in get_plugins()
if plugin["templates_path"]
]
template_paths.extend(plugin_template_paths)
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader,
autoescape=True,
enable_async=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self._register_renderers()
self._permission_checks = collections.deque(maxlen=200)
self._root_token = secrets.token_hex(32)
async def invoke_startup(self):
for hook in pm.hook.startup(datasette=self):
if callable(hook):
hook = hook()
if asyncio.iscoroutine(hook):
hook = await hook
def sign(self, value, namespace="default"):
return URLSafeSerializer(self._secret, namespace).dumps(value)
def unsign(self, signed, namespace="default"):
return URLSafeSerializer(self._secret, namespace).loads(signed)
def get_database(self, name=None):
if name is None:
return next(iter(self.databases.values()))
return self.databases[name]
def add_database(self, name, db):
self.databases[name] = db
def remove_database(self, name):
self.databases.pop(name)
def config(self, key):
return self._config.get(key, None)
def config_dict(self):
# Returns a fully resolved config dictionary, useful for templates
return {
option.name: self.config(option.name)
for option in CONFIG_OPTIONS
}
def metadata(self, key=None, database=None, table=None, fallback=True):
"""
Looks up metadata, cascading backwards from specified level.
Returns None if metadata value is not found.
"""
assert not (
database is None and table is not None
), "Cannot call metadata() with table= specified but not database="
databases = self._metadata.get("databases") or {}
search_list = []
if database is not None:
search_list.append(databases.get(database) or {})
if table is not None:
table_metadata = ((databases.get(database) or {}).get("tables")
or {}).get(table) or {}
search_list.insert(0, table_metadata)
search_list.append(self._metadata)
if not fallback:
# No fallback allowed, so just use the first one in the list
search_list = search_list[:1]
if key is not None:
for item in search_list:
if key in item:
return item[key]
return None
else:
# Return the merged list
m = {}
for item in search_list:
m.update(item)
return m
def plugin_config(self,
plugin_name,
database=None,
table=None,
fallback=True):
"Return config for plugin, falling back from specified database/table"
plugins = self.metadata("plugins",
database=database,
table=table,
fallback=fallback)
if plugins is None:
return None
plugin_config = plugins.get(plugin_name)
# Resolve any $file and $env keys
plugin_config = resolve_env_secrets(plugin_config, os.environ)
return plugin_config
def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
self._app_css_hash = hashlib.sha1(
open(os.path.join(str(app_root),
"datasette/static/app.css")).read().encode(
"utf8")).hexdigest()[:6]
return self._app_css_hash
async def get_canned_queries(self, database_name, actor):
queries = self.metadata(
"queries", database=database_name, fallback=False) or {}
for more_queries in pm.hook.canned_queries(
datasette=self,
database=database_name,
actor=actor,
):
if callable(more_queries):
more_queries = more_queries()
if asyncio.iscoroutine(more_queries):
more_queries = await more_queries
queries.update(more_queries or {})
# Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}}
for key in queries:
if not isinstance(queries[key], dict):
queries[key] = {"sql": queries[key]}
# Also make sure "name" is available:
queries[key]["name"] = key
return queries
async def get_canned_query(self, database_name, query_name, actor):
queries = await self.get_canned_queries(database_name, actor)
query = queries.get(query_name)
if query:
return query
def update_with_inherited_metadata(self, metadata):
# Fills in source/license with defaults, if available
metadata.update({
"source":
metadata.get("source") or self.metadata("source"),
"source_url":
metadata.get("source_url") or self.metadata("source_url"),
"license":
metadata.get("license") or self.metadata("license"),
"license_url":
metadata.get("license_url") or self.metadata("license_url"),
"about":
metadata.get("about") or self.metadata("about"),
"about_url":
metadata.get("about_url") or self.metadata("about_url"),
})
def _prepare_connection(self, conn, database):
conn.row_factory = sqlite3.Row
conn.text_factory = lambda x: str(x, "utf-8", "replace")
for name, num_args, func in self.sqlite_functions:
conn.create_function(name, num_args, func)
if self.sqlite_extensions:
conn.enable_load_extension(True)
for extension in self.sqlite_extensions:
conn.execute("SELECT load_extension('{}')".format(extension))
if self.config("cache_size_kb"):
conn.execute("PRAGMA cache_size=-{}".format(
self.config("cache_size_kb")))
# pylint: disable=no-member
pm.hook.prepare_connection(conn=conn,
database=database,
datasette=self)
def add_message(self, request, message, type=INFO):
if not hasattr(request, "_messages"):
request._messages = []
request._messages_should_clear = False
request._messages.append((message, type))
def _write_messages_to_response(self, request, response):
if getattr(request, "_messages", None):
# Set those messages
response.set_cookie("ds_messages",
self.sign(request._messages, "messages"))
elif getattr(request, "_messages_should_clear", False):
response.set_cookie("ds_messages", "", expires=0, max_age=0)
def _show_messages(self, request):
if getattr(request, "_messages", None):
request._messages_should_clear = True
messages = request._messages
request._messages = []
return messages
else:
return []
async def permission_allowed(self,
actor,
action,
resource=None,
default=False):
"Check permissions using the permissions_allowed plugin hook"
result = None
for check in pm.hook.permission_allowed(
datasette=self,
actor=actor,
action=action,
resource=resource,
):
if callable(check):
check = check()
if asyncio.iscoroutine(check):
check = await check
if check is not None:
result = check
used_default = False
if result is None:
result = default
used_default = True
self._permission_checks.append({
"when":
datetime.datetime.utcnow().isoformat(),
"actor":
actor,
"action":
action,
"resource":
resource,
"used_default":
used_default,
"result":
result,
})
return result
async def execute(
self,
db_name,
sql,
params=None,
truncate=False,
custom_time_limit=None,
page_size=None,
log_sql_errors=True,
):
return await self.databases[db_name].execute(
sql,
params=params,
truncate=truncate,
custom_time_limit=custom_time_limit,
page_size=page_size,
log_sql_errors=log_sql_errors,
)
async def expand_foreign_keys(self, database, table, column, values):
"Returns dict mapping (column, value) -> label"
labeled_fks = {}
db = self.databases[database]
foreign_keys = await db.foreign_keys_for_table(table)
# Find the foreign_key for this column
try:
fk = [
foreign_key for foreign_key in foreign_keys
if foreign_key["column"] == column
][0]
except IndexError:
return {}
label_column = await db.label_column_for_table(fk["other_table"])
if not label_column:
return {(fk["column"], value): str(value) for value in values}
labeled_fks = {}
sql = """
select {other_column}, {label_column}
from {other_table}
where {other_column} in ({placeholders})
""".format(
other_column=escape_sqlite(fk["other_column"]),
label_column=escape_sqlite(label_column),
other_table=escape_sqlite(fk["other_table"]),
placeholders=", ".join(["?"] * len(set(values))),
)
try:
results = await self.execute(database, sql, list(set(values)))
except QueryInterrupted:
pass
else:
for id, value in results:
labeled_fks[(fk["column"], id)] = value
return labeled_fks
def absolute_url(self, request, path):
url = urllib.parse.urljoin(request.url, path)
if url.startswith("http://") and self.config("force_https_urls"):
url = "https://" + url[len("http://"):]
return url
def _register_custom_units(self):
"Register any custom units defined in the metadata.json with Pint"
for unit in self.metadata("custom_units") or []:
ureg.define(unit)
def _connected_databases(self):
return [{
"name": d.name,
"path": d.path,
"size": d.size,
"is_mutable": d.is_mutable,
"is_memory": d.is_memory,
"hash": d.hash,
} for d in sorted(self.databases.values(), key=lambda d: d.name)]
def _versions(self):
conn = sqlite3.connect(":memory:")
self._prepare_connection(conn, ":memory:")
sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
sqlite_extensions = {}
for extension, testsql, hasversion in (
("json1", "SELECT json('{}')", False),
("spatialite", "SELECT spatialite_version()", True),
):
try:
result = conn.execute(testsql)
if hasversion:
sqlite_extensions[extension] = result.fetchone()[0]
else:
sqlite_extensions[extension] = None
except Exception:
pass
# Figure out supported FTS versions
fts_versions = []
for fts in ("FTS5", "FTS4", "FTS3"):
try:
conn.execute(
"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(
fts=fts))
fts_versions.append(fts)
except sqlite3.OperationalError:
continue
datasette_version = {"version": __version__}
if self.version_note:
datasette_version["note"] = self.version_note
return {
"python": {
"version": ".".join(map(str, sys.version_info[:3])),
"full": sys.version,
},
"datasette": datasette_version,
"asgi": "3.0",
"uvicorn": uvicorn.__version__,
"sqlite": {
"version":
sqlite_version,
"fts_versions":
fts_versions,
"extensions":
sqlite_extensions,
"compile_options": [
r[0] for r in conn.execute(
"pragma compile_options;").fetchall()
],
},
}
def _plugins(self, request=None, all=False):
ps = list(get_plugins())
should_show_all = False
if request is not None:
should_show_all = request.args.get("all")
else:
should_show_all = all
if not should_show_all:
ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS]
return [{
"name": p["name"],
"static": p["static_path"] is not None,
"templates": p["templates_path"] is not None,
"version": p.get("version"),
"hooks": p["hooks"],
} for p in ps]
def _threads(self):
threads = list(threading.enumerate())
d = {
"num_threads":
len(threads),
"threads": [{
"name": t.name,
"ident": t.ident,
"daemon": t.daemon
} for t in threads],
}
# Only available in Python 3.7+
if hasattr(asyncio, "all_tasks"):
tasks = asyncio.all_tasks()
d.update({
"num_tasks": len(tasks),
"tasks": [_cleaner_task_str(t) for t in tasks],
})
return d
def _actor(self, request):
return {"actor": request.actor}
def table_metadata(self, database, table):
"Fetch table-specific metadata."
return ((self.metadata("databases")
or {}).get(database, {}).get("tables", {}).get(table, {}))
def _register_renderers(self):
""" Register output renderers which output data in custom formats. """
# Built-in renderers
self.renderers["json"] = (json_renderer, lambda: True)
# Hooks
hook_renderers = []
# pylint: disable=no-member
for hook in pm.hook.register_output_renderer(datasette=self):
if type(hook) == list:
hook_renderers += hook
else:
hook_renderers.append(hook)
for renderer in hook_renderers:
self.renderers[renderer["extension"]] = (
# It used to be called "callback" - remove this in Datasette 1.0
renderer.get("render") or renderer["callback"],
renderer.get("can_render") or (lambda: True),
)
async def render_template(self,
templates,
context=None,
request=None,
view_name=None):
context = context or {}
if isinstance(templates, Template):
template = templates
else:
if isinstance(templates, str):
templates = [templates]
template = self.jinja_env.select_template(templates)
body_scripts = []
# pylint: disable=no-member
for script in pm.hook.extra_body_script(
template=template.name,
database=context.get("database"),
table=context.get("table"),
view_name=view_name,
datasette=self,
):
body_scripts.append(Markup(script))
extra_template_vars = {}
# pylint: disable=no-member
for extra_vars in pm.hook.extra_template_vars(
template=template.name,
database=context.get("database"),
table=context.get("table"),
view_name=view_name,
request=request,
datasette=self,
):
if callable(extra_vars):
extra_vars = extra_vars()
if asyncio.iscoroutine(extra_vars):
extra_vars = await extra_vars
assert isinstance(extra_vars,
dict), "extra_vars is of type {}".format(
type(extra_vars))
extra_template_vars.update(extra_vars)
template_context = {
**context,
**{
"app_css_hash":
self.app_css_hash(),
"zip":
zip,
"body_scripts":
body_scripts,
"format_bytes":
format_bytes,
"show_messages":
lambda: self._show_messages(request),
"extra_css_urls":
self._asset_urls("extra_css_urls", template, context),
"extra_js_urls":
self._asset_urls("extra_js_urls", template, context),
"base_url":
self.config("base_url"),
"csrftoken":
request.scope["csrftoken"] if request else lambda: "",
},
**extra_template_vars,
}
if request and request.args.get("_context") and self.config(
"template_debug"):
return "<pre>{}</pre>".format(
jinja2.escape(
json.dumps(template_context, default=repr, indent=4)))
return await template.render_async(template_context)
def _asset_urls(self, key, template, context):
# Flatten list-of-lists from plugins:
seen_urls = set()
for url_or_dict in itertools.chain(
itertools.chain.from_iterable(
getattr(pm.hook, key)(
template=template.name,
database=context.get("database"),
table=context.get("table"),
datasette=self,
)),
(self.metadata(key) or []),
):
if isinstance(url_or_dict, dict):
url = url_or_dict["url"]
sri = url_or_dict.get("sri")
else:
url = url_or_dict
sri = None
if url in seen_urls:
continue
seen_urls.add(url)
if sri:
yield {"url": url, "sri": sri}
else:
yield {"url": url}
def app(self):
"Returns an ASGI app function that serves the whole of Datasette"
routes = []
for routes_to_add in pm.hook.register_routes():
for regex, view_fn in routes_to_add:
routes.append((regex, wrap_view(view_fn, self)))
def add_route(view, regex):
routes.append((regex, view))
# Generate a regex snippet to match all registered renderer file extensions
renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
add_route(IndexView.as_view(self), r"/(?P<as_format>(\.jsono?)?$)")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
add_route(favicon, "/favicon.ico")
add_route(asgi_static(app_root / "datasette" / "static"),
r"/-/static/(?P<path>.*)$")
for path, dirname in self.static_mounts:
add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$")
# Mount any plugin static/ directories
for plugin in get_plugins():
if plugin["static_path"]:
add_route(
asgi_static(plugin["static_path"]),
"/-/static-plugins/{}/(?P<path>.*)$".format(
plugin["name"]),
)
# Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611
add_route(
asgi_static(plugin["static_path"]),
"/-/static-plugins/{}/(?P<path>.*)$".format(
plugin["name"].replace("-", "_")),
)
add_route(
JsonDataView.as_view(self, "metadata.json",
lambda: self._metadata),
r"/-/metadata(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self, "versions.json", self._versions),
r"/-/versions(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self,
"plugins.json",
self._plugins,
needs_request=True),
r"/-/plugins(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self, "config.json", lambda: self._config),
r"/-/config(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self, "threads.json", self._threads),
r"/-/threads(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self, "databases.json",
self._connected_databases),
r"/-/databases(?P<as_format>(\.json)?)$",
)
add_route(
JsonDataView.as_view(self,
"actor.json",
self._actor,
needs_request=True),
r"/-/actor(?P<as_format>(\.json)?)$",
)
add_route(
AuthTokenView.as_view(self),
r"/-/auth-token$",
)
add_route(
LogoutView.as_view(self),
r"/-/logout$",
)
add_route(
PermissionsDebugView.as_view(self),
r"/-/permissions$",
)
add_route(
MessagesDebugView.as_view(self),
r"/-/messages$",
)
add_route(
PatternPortfolioView.as_view(self),
r"/-/patterns$",
)
add_route(DatabaseDownload.as_view(self),
r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$")
add_route(
DatabaseView.as_view(self),
r"/(?P<db_name>[^/]+?)(?P<as_format>" + renderer_regex +
r"|.jsono|\.csv)?$",
)
add_route(
TableView.as_view(self),
r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)",
)
add_route(
RowView.as_view(self),
r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>"
+ renderer_regex + r")?$",
)
self._register_custom_units()
async def setup_db():
# First time server starts up, calculate table counts for immutable databases
for dbname, database in self.databases.items():
if not database.is_mutable:
await database.table_counts(limit=60 * 60 * 1000)
asgi = AsgiLifespan(
AsgiTracer(
asgi_csrf.asgi_csrf(
DatasetteRouter(self, routes),
signing_secret=self._secret,
cookie_name="ds_csrftoken",
)),
on_startup=setup_db,
)
for wrapper in pm.hook.asgi_wrapper(datasette=self):
asgi = wrapper(asgi)
return asgi
class Datasette:
def __init__(
self,
files,
immutables=None,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
version_note=None,
):
immutables = immutables or []
self.files = tuple(files) + tuple(immutables)
self.immutables = set(immutables)
if not self.files:
self.files = [MEMORY]
elif memory:
self.files = (MEMORY, ) + self.files
self.databases = {}
self.inspect_data = inspect_data
for file in self.files:
path = file
is_memory = False
if file is MEMORY:
path = None
is_memory = True
is_mutable = path not in self.immutables
db = ConnectedDatabase(self,
path,
is_mutable=is_mutable,
is_memory=is_memory)
if db.name in self.databases:
raise Exception("Multiple files with same stem: {}".format(
db.name))
self.databases[db.name] = db
self.cache_headers = cache_headers
self.cors = cors
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
self.template_dir = template_dir
self.plugins_dir = plugins_dir
self.static_mounts = static_mounts or []
self._config = dict(DEFAULT_CONFIG, **(config or {}))
self.renderers = {} # File extension -> renderer function
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.config("num_sql_threads"))
self.max_returned_rows = self.config("max_returned_rows")
self.sql_time_limit_ms = self.config("sql_time_limit_ms")
self.page_size = self.config("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
async def run_sanity_checks(self):
# Only one check right now, for Spatialite
for database_name, database in self.databases.items():
# Run pragma_info on every table
for table in await database.table_names():
try:
await self.execute(
database_name,
"PRAGMA table_info({});".format(escape_sqlite(table)),
)
except sqlite3.OperationalError as e:
if e.args[0] == "no such module: VirtualSpatialIndex":
raise click.UsageError(
"It looks like you're trying to load a SpatiaLite"
" database without first loading the SpatiaLite module."
"\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html"
)
else:
raise
def config(self, key):
return self._config.get(key, None)
def config_dict(self):
# Returns a fully resolved config dictionary, useful for templates
return {
option.name: self.config(option.name)
for option in CONFIG_OPTIONS
}
def metadata(self, key=None, database=None, table=None, fallback=True):
"""
Looks up metadata, cascading backwards from specified level.
Returns None if metadata value is not found.
"""
assert not (
database is None and table is not None
), "Cannot call metadata() with table= specified but not database="
databases = self._metadata.get("databases") or {}
search_list = []
if database is not None:
search_list.append(databases.get(database) or {})
if table is not None:
table_metadata = ((databases.get(database) or {}).get("tables")
or {}).get(table) or {}
search_list.insert(0, table_metadata)
search_list.append(self._metadata)
if not fallback:
# No fallback allowed, so just use the first one in the list
search_list = search_list[:1]
if key is not None:
for item in search_list:
if key in item:
return item[key]
return None
else:
# Return the merged list
m = {}
for item in search_list:
m.update(item)
return m
def plugin_config(self,
plugin_name,
database=None,
table=None,
fallback=True):
"Return config for plugin, falling back from specified database/table"
plugins = self.metadata("plugins",
database=database,
table=table,
fallback=fallback)
if plugins is None:
return None
return plugins.get(plugin_name)
def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
self._app_css_hash = hashlib.sha1(
open(os.path.join(str(app_root),
"datasette/static/app.css")).read().encode(
"utf8")).hexdigest()[:6]
return self._app_css_hash
def get_canned_queries(self, database_name):
queries = self.metadata(
"queries", database=database_name, fallback=False) or {}
names = queries.keys()
return [self.get_canned_query(database_name, name) for name in names]
def get_canned_query(self, database_name, query_name):
queries = self.metadata(
"queries", database=database_name, fallback=False) or {}
query = queries.get(query_name)
if query:
if not isinstance(query, dict):
query = {"sql": query}
query["name"] = query_name
return query
async def get_table_definition(self, database_name, table, type_="table"):
table_definition_rows = list(await self.execute(
database_name,
"select sql from sqlite_master where name = :n and type=:t",
{
"n": table,
"t": type_
},
))
if not table_definition_rows:
return None
return table_definition_rows[0][0]
def get_view_definition(self, database_name, view):
return self.get_table_definition(database_name, view, "view")
def update_with_inherited_metadata(self, metadata):
# Fills in source/license with defaults, if available
metadata.update({
"source":
metadata.get("source") or self.metadata("source"),
"source_url":
metadata.get("source_url") or self.metadata("source_url"),
"license":
metadata.get("license") or self.metadata("license"),
"license_url":
metadata.get("license_url") or self.metadata("license_url"),
"about":
metadata.get("about") or self.metadata("about"),
"about_url":
metadata.get("about_url") or self.metadata("about_url"),
})
def prepare_connection(self, conn):
conn.row_factory = sqlite3.Row
conn.text_factory = lambda x: str(x, "utf-8", "replace")
for name, num_args, func in self.sqlite_functions:
conn.create_function(name, num_args, func)
if self.sqlite_extensions:
conn.enable_load_extension(True)
for extension in self.sqlite_extensions:
conn.execute("SELECT load_extension('{}')".format(extension))
if self.config("cache_size_kb"):
conn.execute("PRAGMA cache_size=-{}".format(
self.config("cache_size_kb")))
# pylint: disable=no-member
pm.hook.prepare_connection(conn=conn)
async def table_exists(self, database, table):
results = await self.execute(
database,
"select 1 from sqlite_master where type='table' and name=?",
params=(table, ),
)
return bool(results.rows)
async def expand_foreign_keys(self, database, table, column, values):
"Returns dict mapping (column, value) -> label"
labeled_fks = {}
foreign_keys = await self.foreign_keys_for_table(database, table)
# Find the foreign_key for this column
try:
fk = [
foreign_key for foreign_key in foreign_keys
if foreign_key["column"] == column
][0]
except IndexError:
return {}
label_column = await self.label_column_for_table(
database, fk["other_table"])
if not label_column:
return {(fk["column"], value): str(value) for value in values}
labeled_fks = {}
sql = """
select {other_column}, {label_column}
from {other_table}
where {other_column} in ({placeholders})
""".format(
other_column=escape_sqlite(fk["other_column"]),
label_column=escape_sqlite(label_column),
other_table=escape_sqlite(fk["other_table"]),
placeholders=", ".join(["?"] * len(set(values))),
)
try:
results = await self.execute(database, sql, list(set(values)))
except InterruptedError:
pass
else:
for id, value in results:
labeled_fks[(fk["column"], id)] = value
return labeled_fks
def absolute_url(self, request, path):
url = urllib.parse.urljoin(request.url, path)
if url.startswith("http://") and self.config("force_https_urls"):
url = "https://" + url[len("http://"):]
return url
def register_custom_units(self):
"Register any custom units defined in the metadata.json with Pint"
for unit in self.metadata("custom_units") or []:
ureg.define(unit)
def connected_databases(self):
return [{
"name": d.name,
"path": d.path,
"size": d.size,
"is_mutable": d.is_mutable,
"is_memory": d.is_memory,
"hash": d.hash,
} for d in sorted(self.databases.values(), key=lambda d: d.name)]
def versions(self):
conn = sqlite3.connect(":memory:")
self.prepare_connection(conn)
sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
sqlite_extensions = {}
for extension, testsql, hasversion in (
("json1", "SELECT json('{}')", False),
("spatialite", "SELECT spatialite_version()", True),
):
try:
result = conn.execute(testsql)
if hasversion:
sqlite_extensions[extension] = result.fetchone()[0]
else:
sqlite_extensions[extension] = None
except Exception:
pass
# Figure out supported FTS versions
fts_versions = []
for fts in ("FTS5", "FTS4", "FTS3"):
try:
conn.execute(
"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(
fts=fts))
fts_versions.append(fts)
except sqlite3.OperationalError:
continue
datasette_version = {"version": __version__}
if self.version_note:
datasette_version["note"] = self.version_note
return {
"python": {
"version": ".".join(map(str, sys.version_info[:3])),
"full": sys.version,
},
"datasette": datasette_version,
"sqlite": {
"version":
sqlite_version,
"fts_versions":
fts_versions,
"extensions":
sqlite_extensions,
"compile_options": [
r[0] for r in conn.execute(
"pragma compile_options;").fetchall()
],
},
}
def plugins(self, show_all=False):
ps = list(get_plugins(pm))
if not show_all:
ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS]
return [{
"name": p["name"],
"static": p["static_path"] is not None,
"templates": p["templates_path"] is not None,
"version": p.get("version"),
} for p in ps]
def table_metadata(self, database, table):
"Fetch table-specific metadata."
return ((self.metadata("databases")
or {}).get(database, {}).get("tables", {}).get(table, {}))
async def table_columns(self, db_name, table):
return await self.execute_against_connection_in_thread(
db_name, lambda conn: table_columns(conn, table))
async def foreign_keys_for_table(self, database, table):
return await self.execute_against_connection_in_thread(
database, lambda conn: get_outbound_foreign_keys(conn, table))
async def label_column_for_table(self, db_name, table):
explicit_label_column = self.table_metadata(db_name,
table).get("label_column")
if explicit_label_column:
return explicit_label_column
# If a table has two columns, one of which is ID, then label_column is the other one
column_names = await self.table_columns(db_name, table)
if column_names and len(column_names) == 2 and "id" in column_names:
return [c for c in column_names if c != "id"][0]
# Couldn't find a label:
return None
async def execute_against_connection_in_thread(self, db_name, fn):
def in_thread():
conn = getattr(connections, db_name, None)
if not conn:
db = self.databases[db_name]
if db.is_memory:
conn = sqlite3.connect(":memory:")
else:
# mode=ro or immutable=1?
if db.is_mutable:
qs = "mode=ro"
else:
qs = "immutable=1"
conn = sqlite3.connect(
"file:{}?{}".format(db.path, qs),
uri=True,
check_same_thread=False,
)
self.prepare_connection(conn)
setattr(connections, db_name, conn)
return fn(conn)
return await asyncio.get_event_loop().run_in_executor(
self.executor, in_thread)
async def execute(
self,
db_name,
sql,
params=None,
truncate=False,
custom_time_limit=None,
page_size=None,
log_sql_errors=True,
):
"""Executes sql against db_name in a thread"""
page_size = page_size or self.page_size
def sql_operation_in_thread(conn):
time_limit_ms = self.sql_time_limit_ms
if custom_time_limit and custom_time_limit < time_limit_ms:
time_limit_ms = custom_time_limit
with sqlite_timelimit(conn, time_limit_ms):
try:
cursor = conn.cursor()
cursor.execute(sql, params or {})
max_returned_rows = self.max_returned_rows
if max_returned_rows == page_size:
max_returned_rows += 1
if max_returned_rows and truncate:
rows = cursor.fetchmany(max_returned_rows + 1)
truncated = len(rows) > max_returned_rows
rows = rows[:max_returned_rows]
else:
rows = cursor.fetchall()
truncated = False
except sqlite3.OperationalError as e:
if e.args == ("interrupted", ):
raise InterruptedError(e, sql, params)
if log_sql_errors:
print(
"ERROR: conn={}, sql = {}, params = {}: {}".format(
conn, repr(sql), params, e))
raise
if truncate:
return Results(rows, truncated, cursor.description)
else:
return Results(rows, False, cursor.description)
with trace("sql", database=db_name, sql=sql.strip(), params=params):
results = await self.execute_against_connection_in_thread(
db_name, sql_operation_in_thread)
return results
def register_renderers(self):
""" Register output renderers which output data in custom formats. """
# Built-in renderers
self.renderers["json"] = json_renderer
# Hooks
hook_renderers = []
for hook in pm.hook.register_output_renderer(datasette=self):
if type(hook) == list:
hook_renderers += hook
else:
hook_renderers.append(hook)
for renderer in hook_renderers:
self.renderers[renderer["extension"]] = renderer["callback"]
def app(self):
class TracingSanic(Sanic):
async def handle_request(self, request, write_callback,
stream_callback):
if request.args.get("_trace"):
request["traces"] = []
request["trace_start"] = time.time()
with capture_traces(request["traces"]):
await super().handle_request(request, write_callback,
stream_callback)
else:
await super().handle_request(request, write_callback,
stream_callback)
app = TracingSanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
# pylint: disable=no-member
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
self.register_renderers()
# Generate a regex snippet to match all registered renderer file extensions
renderer_regex = "|".join(r"\." + key for key in self.renderers.keys())
app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "metadata.json",
lambda: self._metadata),
r"/-/metadata<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
r"/-/versions<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
r"/-/plugins<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self._config),
r"/-/config<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "databases.json",
self.connected_databases),
r"/-/databases<as_format:(\.json)?$>",
)
app.add_route(DatabaseDownload.as_view(self),
r"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(
DatabaseView.as_view(self),
r"/<db_name:[^/]+?><as_format:(" + renderer_regex +
r"|.jsono|\.csv)?$>",
)
app.add_route(TableView.as_view(self),
r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>")
app.add_route(
RowView.as_view(self),
r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(" +
renderer_regex + r")?$>",
)
self.register_custom_units()
# On 404 with a trailing slash redirect to path without that slash:
# pylint: disable=unused-variable
@app.middleware("response")
def redirect_on_404_with_trailing_slash(request, original_response):
if original_response.status == 404 and request.path.endswith("/"):
path = request.path.rstrip("/")
if request.query_string:
path = "{}?{}".format(path, request.query_string)
return response.redirect(path)
@app.middleware("response")
async def add_traces_to_response(request, response):
if request.get("traces") is None:
return
traces = request["traces"]
trace_info = {
"request_duration_ms":
1000 * (time.time() - request["trace_start"]),
"sum_trace_duration_ms":
sum(t["duration_ms"] for t in traces),
"num_traces":
len(traces),
"traces":
traces,
}
if "text/html" in response.content_type and b"</body>" in response.body:
extra = json.dumps(trace_info, indent=2)
extra_html = "<pre>{}</pre></body>".format(extra).encode(
"utf8")
response.body = response.body.replace(b"</body>", extra_html)
elif "json" in response.content_type and response.body.startswith(
b"{"):
data = json.loads(response.body.decode("utf8"))
if "_trace" not in data:
data["_trace"] = trace_info
response.body = json.dumps(data).encode("utf8")
@app.exception(Exception)
def on_exception(request, exception):
title = None
help = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request is not None and request.path.split("?")[0].endswith(
".json"):
r = response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
r = response.html(template.render(info), status=status)
if self.cors:
r.headers["Access-Control-Allow-Origin"] = "*"
return r
# First time server starts up, calculate table counts for immutable databases
@app.listener("before_server_start")
async def setup_db(app, loop):
for dbname, database in self.databases.items():
if not database.is_mutable:
await database.table_counts(limit=60 * 60 * 1000)
return app
class AbstractGenerator(ABC, metaclass=GeneratorMeta):
"""
Abstract base class for code generators based on Jinja2 template engine.
:param schema: the source or the instance of the XSD schema.
:param searchpath: additional search path for custom templates. \
If provided the search path has priority over searchpaths defined \
in generator class.
:param types_map: a dictionary with custom mapping for XSD types.
"""
formal_language = None
"""The formal language associated to the code generator (eg. Python)."""
searchpaths = None
"""
Directory paths for searching templates, specified with a list or a tuple.
Each path must be provided as relative from the directory of the module
where the class is defined. Extends the searchpath defined in base classes.
"""
builtin_types = {
'anyType': '',
'anySimpleType': '',
}
"""
Translation map for XSD builtin types. Updates the builtin_types
defined in base classes.
"""
def __init__(self, schema, searchpath=None, types_map=None):
if isinstance(schema, xmlschema.XMLSchemaBase):
self.schema = schema
else:
self.schema = xmlschema.XMLSchema11(schema)
file_loaders = []
if searchpath:
file_loaders.append(FileSystemLoader(searchpath))
if self.searchpaths:
file_loaders.extend(
FileSystemLoader(str(path))
for path in reversed(self.searchpaths))
if not file_loaders:
raise ValueError("no search paths defined!")
loader = ChoiceLoader(
file_loaders) if len(file_loaders) > 1 else file_loaders[0]
self.types_map = self.builtin_types.copy()
if types_map:
if not self.schema.target_namespace:
self.types_map.update(types_map)
else:
ns_part = '{%s}' % self.schema.target_namespace
self.types_map.update(
(ns_part + k, v) for k, v in types_map.items())
self.filters = {}
self.tests = {}
for name in filter(lambda x: callable(getattr(self, x)), dir(self)):
method = getattr(self, name)
if inspect.isfunction(method):
# static methods
if getattr(method, 'is_filter', False):
self.filters[name] = method
elif getattr(method, 'is_test', False):
self.tests[name] = method
elif inspect.isroutine(method) and hasattr(method, '__func__'):
# class and instance methods
if getattr(method.__func__, 'is_filter', False):
self.filters[name] = method
elif getattr(method.__func__, 'is_test', False):
self.tests[name] = method
type_mapping_filter = '{}_type'.format(
self.formal_language).lower().replace(' ', '_')
if type_mapping_filter not in self.filters:
self.filters[type_mapping_filter] = self.map_type
self._env = Environment(loader=loader)
self._env.filters.update(self.filters)
self._env.tests.update(self.tests)
def __repr__(self):
if self.schema.url:
return '%s(schema=%r)' % (self.__class__.__name__,
self.schema.name)
return '%s(namespace=%r)' % (self.__class__.__name__,
self.schema.target_namespace)
def list_templates(self, extensions=None, filter_func=None):
return self._env.list_templates(extensions, filter_func)
def matching_templates(self, name):
return self._env.list_templates(filter_func=lambda x: fnmatch(x, name))
def get_template(self, name, parent=None, global_vars=None):
return self._env.get_template(name, parent, global_vars)
def select_template(self, names, parent=None, global_vars=None):
return self._env.select_template(names, parent, global_vars)
def render(self, names, parent=None, global_vars=None):
if isinstance(names, str):
names = [names]
elif not all(isinstance(x, str) for x in names):
raise TypeError("'names' argument must contain only strings!")
results = []
for name in names:
try:
template = self._env.get_template(name, parent, global_vars)
except TemplateNotFound as err:
logger.debug("name %r: %s", name, str(err))
except TemplateAssertionError as err:
logger.warning("template %r: %s", name, str(err))
else:
results.append(template.render(schema=self.schema))
return results
def render_to_files(self,
names,
parent=None,
global_vars=None,
output_dir='.',
force=False):
if isinstance(names, str):
names = [names]
elif not all(isinstance(x, str) for x in names):
raise TypeError("'names' argument must contain only strings!")
template_names = []
for name in names:
if is_shell_wildcard(name):
template_names.extend(self.matching_templates(name))
else:
template_names.append(name)
output_dir = Path(output_dir)
rendered = []
for name in template_names:
try:
template = self._env.get_template(name, parent, global_vars)
except TemplateNotFound as err:
logger.debug("name %r: %s", name, str(err))
except TemplateAssertionError as err:
logger.warning("template %r: %s", name, str(err))
else:
output_file = output_dir.joinpath(
Path(name).name).with_suffix('')
if not force and output_file.exists():
continue
result = template.render(schema=self.schema)
logger.info("write file %r", str(output_file))
with open(output_file, 'w') as fp:
fp.write(result)
rendered.append(template.filename)
return rendered
def map_type(self, obj):
"""
Maps an XSD type to a type declaration of the target language.
This method is registered as filter with a name dependant from
the language name (eg. c_type).
:param obj: an XSD type or another type-related declaration as \
an attribute or an element.
:return: an empty string for non-XSD objects.
"""
if isinstance(obj, XsdType):
xsd_type = obj
elif isinstance(obj, (XsdAttribute, XsdElement)):
xsd_type = obj.type
else:
return ''
try:
return self.types_map[xsd_type.name]
except KeyError:
try:
return self.types_map[xsd_type.base_type.name]
except (KeyError, AttributeError):
if xsd_type.is_complex():
return self.types_map[xsd_qname('anyType')]
else:
return self.types_map[xsd_qname('anySimpleType')]
@staticmethod
@filter_method
def name(obj, unnamed='none'):
"""
Get the unqualified name of the provided object. Invalid
chars for identifiers are replaced by an underscore.
:param obj: an XSD object or a named object or a string.
:param unnamed: value for unnamed objects. Defaults to 'none'.
:return: str
"""
try:
name = obj.local_name
except AttributeError:
try:
obj = obj.name
except AttributeError:
pass
if not isinstance(obj, str):
return unnamed
try:
if obj[0] == '{':
_, name = obj.split('}')
elif ':' in obj:
prefix, name = obj.split(':')
if NCNAME_PATTERN.match(prefix) is None:
return ''
else:
name = obj
except (IndexError, ValueError):
return ''
else:
if not isinstance(name, str):
return ''
if NCNAME_PATTERN.match(name) is None:
return unnamed
return name.replace('.', '_').replace('-', '_')
@filter_method
def qname(self, obj, unnamed='none', sep='__'):
"""
Get the QName of the provided object. Invalid chars for
identifiers are replaced by an underscore.
:param obj: an XSD object or a named object or a string.
:param unnamed: value for unnamed objects. Defaults to 'none'.
:param sep: the replacement for colon. Defaults to double underscore.
:return: str
"""
try:
qname = obj.prefixed_name
except AttributeError:
try:
obj = obj.name
except AttributeError:
pass
if not isinstance(obj, str):
return unnamed
try:
if obj[0] == '{':
namespace, local_name = obj.split('}')
for prefix, uri in self.schema.namespaces.items():
if uri == namespace:
qname = '%s:%s' % (uri, local_name)
break
else:
qname = local_name
else:
qname = obj
except (IndexError, ValueError):
return unnamed
if not qname or QNAME_PATTERN.match(qname) is None:
return unnamed
return qname.replace('.', '_').replace('-', '_').replace(':', sep)
@staticmethod
@filter_method
def namespace(obj):
try:
namespace = obj.target_namespace
except AttributeError:
try:
obj = obj.name
except AttributeError:
pass
try:
if not isinstance(obj, str) or obj[0] != '{':
return ''
namespace, _ = obj.split('}')
except (IndexError, ValueError):
return ''
else:
if not isinstance(namespace, str):
return ''
return namespace
@staticmethod
@filter_method
def type_name(obj, suffix=None, unnamed='none'):
"""
Get the unqualified name of the XSD type. Invalid
chars for identifiers are replaced by an underscore.
:param obj: an instance of (XsdType|XsdAttribute|XsdElement).
:param suffix: force a suffix. For default removes '_type' or 'Type' suffixes.
:param unnamed: value for unnamed XSD types. Defaults to 'none'.
:return: str
"""
if isinstance(obj, XsdType):
name = obj.local_name or unnamed
elif isinstance(obj, (XsdElement, XsdAttribute)):
name = obj.type.local_name or unnamed
else:
name = unnamed
if not name or NCNAME_PATTERN.match(name) is None:
name = unnamed
if name.endswith('Type'):
name = name[:-4]
elif name.endswith('_type'):
name = name[:-5]
if suffix:
name = '{}{}'.format(name, suffix)
return name.replace('.', '_').replace('-', '_')
@staticmethod
@filter_method
def type_qname(obj, suffix=None, unnamed='none', sep='__'):
"""
Get the unqualified name of the XSD type. Invalid
chars for identifiers are replaced by an underscore.
:param obj: an instance of (XsdType|XsdAttribute|XsdElement).
:param suffix: force a suffix. For default removes '_type' or 'Type' suffixes.
:param unnamed: value for unnamed XSD types. Defaults to 'none'.
:param sep: the replacement for colon. Defaults to double underscore.
:return: str
"""
if isinstance(obj, XsdType):
qname = obj.prefixed_name or unnamed
elif isinstance(obj, (XsdElement, XsdAttribute)):
qname = obj.type.prefixed_name or unnamed
else:
qname = unnamed
if not qname or QNAME_PATTERN.match(qname) is None:
qname = unnamed
if qname.endswith('Type'):
qname = qname[:-4]
elif qname.endswith('_type'):
qname = qname[:-5]
if suffix:
qname = '{}{}'.format(qname, suffix)
return qname.replace('.', '_').replace('-', '_').replace(':', sep)
@staticmethod
@filter_method
def sort_types(xsd_types, accept_circularity=False):
"""
Returns a sorted sequence of XSD types. Sorted types can be used to build code declarations.
:param xsd_types: a sequence with XSD types.
:param accept_circularity: if set to `True` circularities are accepted. Defaults to `False`.
:return: a list with ordered types.
"""
if not isinstance(xsd_types, (list, tuple)):
try:
xsd_types = list(xsd_types.values())
except AttributeError:
pass
assert all(isinstance(x, XsdType) for x in xsd_types)
ordered_types = [x for x in xsd_types if x.is_simple()]
ordered_types.extend(x for x in xsd_types
if x.is_complex() and x.has_simple_content())
unordered = {
x: []
for x in xsd_types
if x.is_complex() and not x.has_simple_content()
}
for xsd_type in unordered:
for e in xsd_type.content_type.iter_elements():
if e.type in unordered:
unordered[xsd_type].append(e.type)
while unordered:
deleted = 0
for xsd_type in xsd_types:
if xsd_type in unordered:
if not unordered[xsd_type]:
del unordered[xsd_type]
ordered_types.append(xsd_type)
deleted += 1
for xsd_type in unordered:
unordered[xsd_type] = [
x for x in unordered[xsd_type] if x in unordered
]
if not deleted:
if not accept_circularity:
raise ValueError("Circularity found between {!r}".format(
list(unordered)))
ordered_types.extend(list(unordered))
break
assert len(xsd_types) == len(ordered_types)
return ordered_types
def is_derived(self, xsd_type, *names, derivation=None):
for type_name in names:
if not isinstance(type_name, str) or not type_name:
continue
elif type_name[0] == '{':
if xsd_type.is_derived(self.schema.maps.types[type_name],
derivation):
return True
elif ':' in type_name:
try:
other = self.schema.resolve_qname(type_name)
except xmlschema.XMLSchemaException:
continue
else:
if xsd_type.is_derived(other, derivation):
return True
else:
if xsd_type.is_derived(self.schema.types[type_name],
derivation):
return True
return False
@test_method
def derivation(self, xsd_type, *names):
return self.is_derived(xsd_type, *names)
@test_method
def extension(self, xsd_type, *names):
return self.is_derived(xsd_type, *names, derivation='extension')
@test_method
def restriction(self, xsd_type, *names):
return self.is_derived(xsd_type, *names, derivation='restriction')
@staticmethod
@test_method
def multi_sequence(xsd_type):
if xsd_type.has_simple_content():
return False
return any(e.is_multiple()
for e in xsd_type.content_type.iter_elements())
class Renderer(object):
"""Renders a template."""
def __init__(self, path=None, loader=None, filters=None):
"""Constructs a new Renderer object.
Either path or loader has to be specified.
Keyword arguments:
path -- list or str which represents template locations
loader -- a jinja2 template loader instance (default: None)
filters -- dict containing filters (default: {})
"""
if (path is None and loader is None
or path is not None and loader is not None):
raise ValueError('Either specify path oder loader')
if path is not None:
loader = FileSystemLoader(path)
self._env = Environment(loader=loader)
self._env.add_extension('jinja2.ext.do')
self._add_filters(filters)
def _add_filters(self, filters):
"""Adds new filters to the jinja2 environment.
filters is a dict of filters.
"""
self._env.filters['dateformat'] = dateformat
self._env.filters.update(filters or {})
def _custom_template_names(self, template):
"""Returns a list of custom template names.
template is the name of the original template.
"""
splitted = template.rsplit('/', 1)
name = 'custom_' + splitted[-1]
ret = [name]
if len(splitted) == 2:
ret.append(splitted[0] + '/' + name)
return ret
def render_only(self, template, *args, **kwargs):
"""Renders template template.
The rendered template is returned (no data will be written
or printed).
*args and **kwargs are passed to jinja2 Template's render
method.
"""
names = self._custom_template_names(template)
names.append(template)
tmpl = self._env.select_template(names)
return tmpl.render(*args, **kwargs)
def _render(self, template, out, *args, **kwargs):
"""Renders template template.
out is a file or file-like object to which the rendered
template should be written to.
*args and **kwargs are passed to the render_only method.
"""
text = self.render_only(template, *args, **kwargs)
try:
out.write(text)
except UnicodeEncodeError:
text = text.encode('utf-8')
out.write(text)
def render(self, template, *args, **kwargs):
"""Renders template template.
Writes the rendered template to sys.stdout.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
self._render(template, sys.stdout, *args, **kwargs)
def render_text(self, text, *args, **kwargs):
"""Renders text.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
global TEXT_TEMPLATE
self.render(TEXT_TEMPLATE, text=text, *args, **kwargs)
def render_error(self, template, *args, **kwargs):
"""Renders template template.
Writes the rendered template to sys.stderr.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
self._render(template, sys.stderr, *args, **kwargs)
class Datasette:
def __init__(
self,
files,
num_threads=3,
cache_headers=True,
page_size=100,
max_returned_rows=1000,
sql_time_limit_ms=1000,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
):
self.files = files
self.num_threads = num_threads
self.executor = futures.ThreadPoolExecutor(max_workers=num_threads)
self.cache_headers = cache_headers
self.page_size = page_size
self.max_returned_rows = max_returned_rows
self.sql_time_limit_ms = sql_time_limit_ms
self.cors = cors
self._inspect = inspect_data
self.metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
self.template_dir = template_dir
self.plugins_dir = plugins_dir
self.static_mounts = static_mounts or []
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
self._app_css_hash = hashlib.sha1(
open(os.path.join(str(app_root),
"datasette/static/app.css")).read().encode(
"utf8")).hexdigest()[:6]
return self._app_css_hash
def get_canned_query(self, database_name, query_name):
query = self.metadata.get("databases",
{}).get(database_name,
{}).get("queries",
{}).get(query_name)
if query:
return {"name": query_name, "sql": query}
def asset_urls(self, key):
urls_or_dicts = (self.metadata.get(key) or [])
# Flatten list-of-lists from plugins:
urls_or_dicts += list(
itertools.chain.from_iterable(getattr(pm.hook, key)()))
for url_or_dict in urls_or_dicts:
if isinstance(url_or_dict, dict):
yield {
"url": url_or_dict["url"],
"sri": url_or_dict.get("sri")
}
else:
yield {"url": url_or_dict}
def extra_css_urls(self):
return self.asset_urls("extra_css_urls")
def extra_js_urls(self):
return self.asset_urls("extra_js_urls")
def update_with_inherited_metadata(self, metadata):
# Fills in source/license with defaults, if available
metadata.update({
"source":
metadata.get("source") or self.metadata.get("source"),
"source_url":
metadata.get("source_url") or self.metadata.get("source_url"),
"license":
metadata.get("license") or self.metadata.get("license"),
"license_url":
metadata.get("license_url") or self.metadata.get("license_url"),
})
def prepare_connection(self, conn):
conn.row_factory = sqlite3.Row
conn.text_factory = lambda x: str(x, "utf-8", "replace")
for name, num_args, func in self.sqlite_functions:
conn.create_function(name, num_args, func)
if self.sqlite_extensions:
conn.enable_load_extension(True)
for extension in self.sqlite_extensions:
conn.execute("SELECT load_extension('{}')".format(extension))
pm.hook.prepare_connection(conn=conn)
def inspect(self):
if not self._inspect:
self._inspect = {}
for filename in self.files:
path = Path(filename)
name = path.stem
if name in self._inspect:
raise Exception("Multiple files with same stem %s" % name)
# Calculate hash, efficiently
m = hashlib.sha256()
with path.open("rb") as fp:
while True:
data = fp.read(HASH_BLOCK_SIZE)
if not data:
break
m.update(data)
# List tables and their row counts
database_metadata = self.metadata.get("databases",
{}).get(name, {})
tables = {}
views = []
with sqlite3.connect("file:{}?immutable=1".format(path),
uri=True) as conn:
self.prepare_connection(conn)
table_names = [
r["name"] for r in conn.execute(
'select * from sqlite_master where type="table"')
]
views = [
v[0] for v in conn.execute(
'select name from sqlite_master where type = "view"'
)
]
for table in table_names:
try:
count = conn.execute(
"select count(*) from {}".format(
escape_sqlite(table))).fetchone()[0]
except sqlite3.OperationalError:
# This can happen when running against a FTS virtual tables
# e.g. "select count(*) from some_fts;"
count = 0
# Does this table have a FTS table?
fts_table = detect_fts(conn, table)
# Figure out primary keys
table_info_rows = [
row
for row in conn.execute('PRAGMA table_info("{}")'.
format(table)).fetchall()
if row[-1]
]
table_info_rows.sort(key=lambda row: row[-1])
primary_keys = [str(r[1]) for r in table_info_rows]
label_column = None
# If table has two columns, one of which is ID, then label_column is the other one
column_names = [
r[1] for r in conn.execute(
"PRAGMA table_info({});".format(
escape_sqlite(table))).fetchall()
]
if (column_names and len(column_names) == 2
and "id" in column_names):
label_column = [
c for c in column_names if c != "id"
][0]
table_metadata = database_metadata.get("tables",
{}).get(
table, {})
tables[table] = {
"name": table,
"columns": column_names,
"primary_keys": primary_keys,
"count": count,
"label_column": label_column,
"hidden": table_metadata.get("hidden") or False,
"fts_table": fts_table,
}
foreign_keys = get_all_foreign_keys(conn)
for table, info in foreign_keys.items():
tables[table]["foreign_keys"] = info
# Mark tables 'hidden' if they relate to FTS virtual tables
hidden_tables = [
r["name"] for r in conn.execute("""
select name from sqlite_master
where rootpage = 0
and sql like '%VIRTUAL TABLE%USING FTS%'
""")
]
if detect_spatialite(conn):
# Also hide Spatialite internal tables
hidden_tables += [
"ElementaryGeometries",
"SpatialIndex",
"geometry_columns",
"spatial_ref_sys",
"spatialite_history",
"sql_statements_log",
"sqlite_sequence",
"views_geometry_columns",
"virts_geometry_columns",
] + [
r["name"] for r in conn.execute("""
select name from sqlite_master
where name like "idx_%"
and type = "table"
""")
]
for t in tables.keys():
for hidden_table in hidden_tables:
if t == hidden_table or t.startswith(hidden_table):
tables[t]["hidden"] = True
continue
self._inspect[name] = {
"hash": m.hexdigest(),
"file": str(path),
"tables": tables,
"views": views,
}
return self._inspect
def register_custom_units(self):
"Register any custom units defined in the metadata.json with Pint"
for unit in self.metadata.get("custom_units", []):
ureg.define(unit)
def versions(self):
conn = sqlite3.connect(":memory:")
self.prepare_connection(conn)
sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
sqlite_extensions = {}
for extension, testsql, hasversion in (
("json1", "SELECT json('{}')", False),
("spatialite", "SELECT spatialite_version()", True),
):
try:
result = conn.execute(testsql)
if hasversion:
sqlite_extensions[extension] = result.fetchone()[0]
else:
sqlite_extensions[extension] = None
except Exception as e:
pass
# Figure out supported FTS versions
fts_versions = []
for fts in ("FTS5", "FTS4", "FTS3"):
try:
conn.execute(
"CREATE VIRTUAL TABLE v{fts} USING {fts} (t TEXT)".format(
fts=fts))
fts_versions.append(fts)
except sqlite3.OperationalError:
continue
return {
"python": {
"version": ".".join(map(str, sys.version_info[:3])),
"full": sys.version
},
"datasette": {
"version": __version__
},
"sqlite": {
"version": sqlite_version,
"fts_versions": fts_versions,
"extensions": sqlite_extensions,
},
}
def plugins(self):
return [{
"name": p["name"],
"static": p["static_path"] is not None,
"templates": p["templates_path"] is not None,
"version": p.get("version"),
} for p in get_plugins(pm)]
def app(self):
app = Sanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
app.add_route(IndexView.as_view(self), "/<as_json:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "inspect.json", self.inspect),
"/-/inspect<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "metadata.json", lambda: self.metadata),
"/-/metadata<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
"/-/versions<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
"/-/plugins<as_json:(\.json)?$>",
)
app.add_route(DatabaseView.as_view(self),
"/<db_name:[^/\.]+?><as_json:(\.jsono?)?$>")
app.add_route(DatabaseDownload.as_view(self),
"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(
TableView.as_view(self),
"/<db_name:[^/]+>/<table:[^/]+?><as_json:(\.jsono?)?$>",
)
app.add_route(
RowView.as_view(self),
"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_json:(\.jsono?)?$>",
)
self.register_custom_units()
@app.exception(Exception)
def on_exception(request, exception):
title = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request.path.split("?")[0].endswith(".json"):
return response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
return response.html(template.render(info), status=status)
return app
class Datasette:
def __init__(
self,
files,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
memory=False,
config=None,
version_note=None,
):
self.files = files
if not self.files:
self.files = [MEMORY]
elif memory:
self.files = (MEMORY, ) + self.files
self.cache_headers = cache_headers
self.cors = cors
self._inspect = inspect_data
self._metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
self.template_dir = template_dir
self.plugins_dir = plugins_dir
self.static_mounts = static_mounts or []
self._config = dict(DEFAULT_CONFIG, **(config or {}))
self.version_note = version_note
self.executor = futures.ThreadPoolExecutor(
max_workers=self.config("num_sql_threads"))
self.max_returned_rows = self.config("max_returned_rows")
self.sql_time_limit_ms = self.config("sql_time_limit_ms")
self.page_size = self.config("default_page_size")
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
def config(self, key):
return self._config.get(key, None)
def config_dict(self):
# Returns a fully resolved config dictionary, useful for templates
return {
option.name: self.config(option.name)
for option in CONFIG_OPTIONS
}
def metadata(self, key=None, database=None, table=None, fallback=True):
"""
Looks up metadata, cascading backwards from specified level.
Returns None if metadata value is not found.
"""
assert not (database is None and table is not None), \
"Cannot call metadata() with table= specified but not database="
databases = self._metadata.get("databases") or {}
search_list = []
if database is not None:
search_list.append(databases.get(database) or {})
if table is not None:
table_metadata = ((databases.get(database) or {}).get("tables")
or {}).get(table) or {}
search_list.insert(0, table_metadata)
search_list.append(self._metadata)
if not fallback:
# No fallback allowed, so just use the first one in the list
search_list = search_list[:1]
if key is not None:
for item in search_list:
if key in item:
return item[key]
return None
else:
# Return the merged list
m = {}
for item in search_list:
m.update(item)
return m
def plugin_config(self,
plugin_name,
database=None,
table=None,
fallback=True):
"Return config for plugin, falling back from specified database/table"
plugins = self.metadata("plugins",
database=database,
table=table,
fallback=fallback)
if plugins is None:
return None
return plugins.get(plugin_name)
def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
self._app_css_hash = hashlib.sha1(
open(os.path.join(str(app_root),
"datasette/static/app.css")).read().encode(
"utf8")).hexdigest()[:6]
return self._app_css_hash
def get_canned_queries(self, database_name):
queries = self.metadata(
"queries", database=database_name, fallback=False) or {}
names = queries.keys()
return [self.get_canned_query(database_name, name) for name in names]
def get_canned_query(self, database_name, query_name):
queries = self.metadata(
"queries", database=database_name, fallback=False) or {}
query = queries.get(query_name)
if query:
if not isinstance(query, dict):
query = {"sql": query}
query["name"] = query_name
return query
async def get_table_definition(self, database_name, table, type_="table"):
table_definition_rows = list(await self.execute(
database_name,
'select sql from sqlite_master where name = :n and type=:t',
{
"n": table,
"t": type_
},
))
if not table_definition_rows:
return None
return table_definition_rows[0][0]
def get_view_definition(self, database_name, view):
return self.get_table_definition(database_name, view, 'view')
def update_with_inherited_metadata(self, metadata):
# Fills in source/license with defaults, if available
metadata.update({
"source":
metadata.get("source") or self.metadata("source"),
"source_url":
metadata.get("source_url") or self.metadata("source_url"),
"license":
metadata.get("license") or self.metadata("license"),
"license_url":
metadata.get("license_url") or self.metadata("license_url"),
"about":
metadata.get("about") or self.metadata("about"),
"about_url":
metadata.get("about_url") or self.metadata("about_url"),
})
def prepare_connection(self, conn):
conn.row_factory = sqlite3.Row
conn.text_factory = lambda x: str(x, "utf-8", "replace")
for name, num_args, func in self.sqlite_functions:
conn.create_function(name, num_args, func)
if self.sqlite_extensions:
conn.enable_load_extension(True)
for extension in self.sqlite_extensions:
conn.execute("SELECT load_extension('{}')".format(extension))
if self.config("cache_size_kb"):
conn.execute('PRAGMA cache_size=-{}'.format(
self.config("cache_size_kb")))
pm.hook.prepare_connection(conn=conn)
def table_exists(self, database, table):
return table in self.inspect().get(database, {}).get("tables")
def inspect(self):
" Inspect the database and return a dictionary of table metadata "
if self._inspect:
return self._inspect
self._inspect = {}
for filename in self.files:
if filename is MEMORY:
self._inspect[":memory:"] = {
"hash": "000",
"file": ":memory:",
"size": 0,
"views": {},
"tables": {},
}
else:
path = Path(filename)
name = path.stem
if name in self._inspect:
raise Exception("Multiple files with same stem %s" % name)
try:
with sqlite3.connect("file:{}?immutable=1".format(path),
uri=True) as conn:
self.prepare_connection(conn)
self._inspect[name] = {
"hash":
inspect_hash(path),
"file":
str(path),
"size":
path.stat().st_size,
"views":
inspect_views(conn),
"tables":
inspect_tables(conn, (self.metadata("databases")
or {}).get(name, {}))
}
except sqlite3.OperationalError as e:
if (e.args[0] == 'no such module: VirtualSpatialIndex'):
raise click.UsageError(
"It looks like you're trying to load a SpatiaLite"
" database without first loading the SpatiaLite module."
"\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html"
)
else:
raise
return self._inspect
def register_custom_units(self):
"Register any custom units defined in the metadata.json with Pint"
for unit in self.metadata("custom_units") or []:
ureg.define(unit)
def versions(self):
conn = sqlite3.connect(":memory:")
self.prepare_connection(conn)
sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
sqlite_extensions = {}
for extension, testsql, hasversion in (
("json1", "SELECT json('{}')", False),
("spatialite", "SELECT spatialite_version()", True),
):
try:
result = conn.execute(testsql)
if hasversion:
sqlite_extensions[extension] = result.fetchone()[0]
else:
sqlite_extensions[extension] = None
except Exception as e:
pass
# Figure out supported FTS versions
fts_versions = []
for fts in ("FTS5", "FTS4", "FTS3"):
try:
conn.execute(
"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(
fts=fts))
fts_versions.append(fts)
except sqlite3.OperationalError:
continue
datasette_version = {"version": __version__}
if self.version_note:
datasette_version["note"] = self.version_note
return {
"python": {
"version": ".".join(map(str, sys.version_info[:3])),
"full": sys.version
},
"datasette": datasette_version,
"sqlite": {
"version":
sqlite_version,
"fts_versions":
fts_versions,
"extensions":
sqlite_extensions,
"compile_options": [
r[0] for r in conn.execute(
"pragma compile_options;").fetchall()
],
},
}
def plugins(self, show_all=False):
ps = list(get_plugins(pm))
if not show_all:
ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS]
return [{
"name": p["name"],
"static": p["static_path"] is not None,
"templates": p["templates_path"] is not None,
"version": p.get("version"),
} for p in ps]
async def execute(
self,
db_name,
sql,
params=None,
truncate=False,
custom_time_limit=None,
page_size=None,
):
"""Executes sql against db_name in a thread"""
page_size = page_size or self.page_size
def sql_operation_in_thread():
conn = getattr(connections, db_name, None)
if not conn:
info = self.inspect()[db_name]
if info["file"] == ":memory:":
conn = sqlite3.connect(":memory:")
else:
conn = sqlite3.connect(
"file:{}?immutable=1".format(info["file"]),
uri=True,
check_same_thread=False,
)
self.prepare_connection(conn)
setattr(connections, db_name, conn)
time_limit_ms = self.sql_time_limit_ms
if custom_time_limit and custom_time_limit < time_limit_ms:
time_limit_ms = custom_time_limit
with sqlite_timelimit(conn, time_limit_ms):
try:
cursor = conn.cursor()
cursor.execute(sql, params or {})
max_returned_rows = self.max_returned_rows
if max_returned_rows == page_size:
max_returned_rows += 1
if max_returned_rows and truncate:
rows = cursor.fetchmany(max_returned_rows + 1)
truncated = len(rows) > max_returned_rows
rows = rows[:max_returned_rows]
else:
rows = cursor.fetchall()
truncated = False
except sqlite3.OperationalError as e:
if e.args == ('interrupted', ):
raise InterruptedError(e)
print("ERROR: conn={}, sql = {}, params = {}: {}".format(
conn, repr(sql), params, e))
raise
if truncate:
return Results(rows, truncated, cursor.description)
else:
return Results(rows, False, cursor.description)
return await asyncio.get_event_loop().run_in_executor(
self.executor, sql_operation_in_thread)
def app(self):
app = Sanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
app.add_route(IndexView.as_view(self), r"/<as_format:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "inspect.json", self.inspect),
r"/-/inspect<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "metadata.json",
lambda: self._metadata),
r"/-/metadata<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
r"/-/versions<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
r"/-/plugins<as_format:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self._config),
r"/-/config<as_format:(\.json)?$>",
)
app.add_route(DatabaseDownload.as_view(self),
r"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(DatabaseView.as_view(self),
r"/<db_name:[^/]+?><as_format:(\.jsono?|\.csv)?$>")
app.add_route(
TableView.as_view(self),
r"/<db_name:[^/]+>/<table_and_format:[^/]+?$>",
)
app.add_route(
RowView.as_view(self),
r"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_format:(\.jsono?)?$>",
)
self.register_custom_units()
# On 404 with a trailing slash redirect to path without that slash:
@app.middleware("response")
def redirect_on_404_with_trailing_slash(request, original_response):
if original_response.status == 404 and request.path.endswith("/"):
path = request.path.rstrip("/")
if request.query_string:
path = "{}?{}".format(path, request.query_string)
return response.redirect(path)
@app.exception(Exception)
def on_exception(request, exception):
title = None
help = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
if exception.messagge_is_html:
message = Markup(message)
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request is not None and request.path.split("?")[0].endswith(
".json"):
return response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
return response.html(template.render(info), status=status)
return app
class Datasette:
def __init__(
self,
files,
num_threads=3,
cache_headers=True,
cors=False,
inspect_data=None,
metadata=None,
sqlite_extensions=None,
template_dir=None,
plugins_dir=None,
static_mounts=None,
config=None,
):
self.files = files
self.num_threads = num_threads
self.executor = futures.ThreadPoolExecutor(max_workers=num_threads)
self.cache_headers = cache_headers
self.cors = cors
self._inspect = inspect_data
self.metadata = metadata or {}
self.sqlite_functions = []
self.sqlite_extensions = sqlite_extensions or []
self.template_dir = template_dir
self.plugins_dir = plugins_dir
self.static_mounts = static_mounts or []
self.config = dict(DEFAULT_CONFIG, **(config or {}))
self.max_returned_rows = self.config["max_returned_rows"]
self.sql_time_limit_ms = self.config["sql_time_limit_ms"]
self.page_size = self.config["default_page_size"]
# Execute plugins in constructor, to ensure they are available
# when the rest of `datasette inspect` executes
if self.plugins_dir:
for filename in os.listdir(self.plugins_dir):
filepath = os.path.join(self.plugins_dir, filename)
mod = module_from_path(filepath, name=filename)
try:
pm.register(mod)
except ValueError:
# Plugin already registered
pass
def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
self._app_css_hash = hashlib.sha1(
open(os.path.join(str(app_root),
"datasette/static/app.css")).read().encode(
"utf8")).hexdigest()[:6]
return self._app_css_hash
def get_canned_query(self, database_name, query_name):
query = self.metadata.get("databases",
{}).get(database_name,
{}).get("queries",
{}).get(query_name)
if query:
return {"name": query_name, "sql": query}
def asset_urls(self, key):
urls_or_dicts = (self.metadata.get(key) or [])
# Flatten list-of-lists from plugins:
urls_or_dicts += list(
itertools.chain.from_iterable(getattr(pm.hook, key)()))
for url_or_dict in urls_or_dicts:
if isinstance(url_or_dict, dict):
yield {
"url": url_or_dict["url"],
"sri": url_or_dict.get("sri")
}
else:
yield {"url": url_or_dict}
def extra_css_urls(self):
return self.asset_urls("extra_css_urls")
def extra_js_urls(self):
return self.asset_urls("extra_js_urls")
def update_with_inherited_metadata(self, metadata):
# Fills in source/license with defaults, if available
metadata.update({
"source":
metadata.get("source") or self.metadata.get("source"),
"source_url":
metadata.get("source_url") or self.metadata.get("source_url"),
"license":
metadata.get("license") or self.metadata.get("license"),
"license_url":
metadata.get("license_url") or self.metadata.get("license_url"),
})
def prepare_connection(self, conn):
conn.row_factory = sqlite3.Row
conn.text_factory = lambda x: str(x, "utf-8", "replace")
for name, num_args, func in self.sqlite_functions:
conn.create_function(name, num_args, func)
if self.sqlite_extensions:
conn.enable_load_extension(True)
for extension in self.sqlite_extensions:
conn.execute("SELECT load_extension('{}')".format(extension))
pm.hook.prepare_connection(conn=conn)
def inspect(self):
" Inspect the database and return a dictionary of table metadata "
if self._inspect:
return self._inspect
self._inspect = {}
for filename in self.files:
path = Path(filename)
name = path.stem
if name in self._inspect:
raise Exception("Multiple files with same stem %s" % name)
with sqlite3.connect("file:{}?immutable=1".format(path),
uri=True) as conn:
self.prepare_connection(conn)
self._inspect[name] = {
"hash":
inspect_hash(path),
"file":
str(path),
"views":
inspect_views(conn),
"tables":
inspect_tables(
conn,
self.metadata.get("databases", {}).get(name, {}))
}
return self._inspect
def register_custom_units(self):
"Register any custom units defined in the metadata.json with Pint"
for unit in self.metadata.get("custom_units", []):
ureg.define(unit)
def versions(self):
conn = sqlite3.connect(":memory:")
self.prepare_connection(conn)
sqlite_version = conn.execute("select sqlite_version()").fetchone()[0]
sqlite_extensions = {}
for extension, testsql, hasversion in (
("json1", "SELECT json('{}')", False),
("spatialite", "SELECT spatialite_version()", True),
):
try:
result = conn.execute(testsql)
if hasversion:
sqlite_extensions[extension] = result.fetchone()[0]
else:
sqlite_extensions[extension] = None
except Exception as e:
pass
# Figure out supported FTS versions
fts_versions = []
for fts in ("FTS5", "FTS4", "FTS3"):
try:
conn.execute(
"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(
fts=fts))
fts_versions.append(fts)
except sqlite3.OperationalError:
continue
return {
"python": {
"version": ".".join(map(str, sys.version_info[:3])),
"full": sys.version
},
"datasette": {
"version": __version__
},
"sqlite": {
"version": sqlite_version,
"fts_versions": fts_versions,
"extensions": sqlite_extensions,
},
}
def plugins(self):
return [{
"name": p["name"],
"static": p["static_path"] is not None,
"templates": p["templates_path"] is not None,
"version": p.get("version"),
} for p in get_plugins(pm)]
async def execute(
self,
db_name,
sql,
params=None,
truncate=False,
custom_time_limit=None,
page_size=None,
):
"""Executes sql against db_name in a thread"""
page_size = page_size or self.page_size
def sql_operation_in_thread():
conn = getattr(connections, db_name, None)
if not conn:
info = self.inspect()[db_name]
conn = sqlite3.connect(
"file:{}?immutable=1".format(info["file"]),
uri=True,
check_same_thread=False,
)
self.prepare_connection(conn)
setattr(connections, db_name, conn)
time_limit_ms = self.sql_time_limit_ms
if custom_time_limit and custom_time_limit < time_limit_ms:
time_limit_ms = custom_time_limit
with sqlite_timelimit(conn, time_limit_ms):
try:
cursor = conn.cursor()
cursor.execute(sql, params or {})
max_returned_rows = self.max_returned_rows
if max_returned_rows == page_size:
max_returned_rows += 1
if max_returned_rows and truncate:
rows = cursor.fetchmany(max_returned_rows + 1)
truncated = len(rows) > max_returned_rows
rows = rows[:max_returned_rows]
else:
rows = cursor.fetchall()
truncated = False
except sqlite3.OperationalError as e:
if e.args == ('interrupted', ):
raise InterruptedError(e)
print("ERROR: conn={}, sql = {}, params = {}: {}".format(
conn, repr(sql), params, e))
raise
if truncate:
return Results(rows, truncated, cursor.description)
else:
return Results(rows, False, cursor.description)
return await asyncio.get_event_loop().run_in_executor(
self.executor, sql_operation_in_thread)
def app(self):
app = Sanic(__name__)
default_templates = str(app_root / "datasette" / "templates")
template_paths = []
if self.template_dir:
template_paths.append(self.template_dir)
template_paths.extend([
plugin["templates_path"] for plugin in get_plugins(pm)
if plugin["templates_path"]
])
template_paths.append(default_templates)
template_loader = ChoiceLoader([
FileSystemLoader(template_paths),
# Support {% extends "default:table.html" %}:
PrefixLoader({"default": FileSystemLoader(default_templates)},
delimiter=":"),
])
self.jinja_env = Environment(loader=template_loader, autoescape=True)
self.jinja_env.filters["escape_css_string"] = escape_css_string
self.jinja_env.filters[
"quote_plus"] = lambda u: urllib.parse.quote_plus(u)
self.jinja_env.filters["escape_sqlite"] = escape_sqlite
self.jinja_env.filters["to_css_class"] = to_css_class
pm.hook.prepare_jinja2_environment(env=self.jinja_env)
app.add_route(IndexView.as_view(self), "/<as_json:(\.jsono?)?$>")
# TODO: /favicon.ico and /-/static/ deserve far-future cache expires
app.add_route(favicon, "/favicon.ico")
app.static("/-/static/", str(app_root / "datasette" / "static"))
for path, dirname in self.static_mounts:
app.static(path, dirname)
# Mount any plugin static/ directories
for plugin in get_plugins(pm):
if plugin["static_path"]:
modpath = "/-/static-plugins/{}/".format(plugin["name"])
app.static(modpath, plugin["static_path"])
app.add_route(
JsonDataView.as_view(self, "inspect.json", self.inspect),
"/-/inspect<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "metadata.json", lambda: self.metadata),
"/-/metadata<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "versions.json", self.versions),
"/-/versions<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "plugins.json", self.plugins),
"/-/plugins<as_json:(\.json)?$>",
)
app.add_route(
JsonDataView.as_view(self, "config.json", lambda: self.config),
"/-/config<as_json:(\.json)?$>",
)
app.add_route(DatabaseView.as_view(self),
"/<db_name:[^/\.]+?><as_json:(\.jsono?)?$>")
app.add_route(DatabaseDownload.as_view(self),
"/<db_name:[^/]+?><as_db:(\.db)$>")
app.add_route(
TableView.as_view(self),
"/<db_name:[^/]+>/<table:[^/]+?><as_json:(\.jsono?)?$>",
)
app.add_route(
RowView.as_view(self),
"/<db_name:[^/]+>/<table:[^/]+?>/<pk_path:[^/]+?><as_json:(\.jsono?)?$>",
)
self.register_custom_units()
@app.exception(Exception)
def on_exception(request, exception):
title = None
if isinstance(exception, NotFound):
status = 404
info = {}
message = exception.args[0]
elif isinstance(exception, InvalidUsage):
status = 405
info = {}
message = exception.args[0]
elif isinstance(exception, DatasetteError):
status = exception.status
info = exception.error_dict
message = exception.message
title = exception.title
else:
status = 500
info = {}
message = str(exception)
traceback.print_exc()
templates = ["500.html"]
if status != 500:
templates = ["{}.html".format(status)] + templates
info.update({
"ok": False,
"error": message,
"status": status,
"title": title
})
if request.path.split("?")[0].endswith(".json"):
return response.json(info, status=status)
else:
template = self.jinja_env.select_template(templates)
return response.html(template.render(info), status=status)
return app
class Renderer(object):
"""Renders a template."""
def __init__(self, path=None, loader=None, filters=None):
"""Constructs a new Renderer object.
Either path or loader has to be specified.
Keyword arguments:
path -- list or str which represents template locations
loader -- a jinja2 template loader instance (default: None)
filters -- dict containing filters (default: {})
"""
if (path is None and loader is None
or path is not None and loader is not None):
raise ValueError('Either specify path oder loader')
if path is not None:
loader = FileSystemLoader(path)
self._env = Environment(loader=loader)
self._env.add_extension('jinja2.ext.do')
self._add_filters(filters)
def _add_filters(self, filters):
"""Adds new filters to the jinja2 environment.
filters is a dict of filters.
"""
self._env.filters['dateformat'] = dateformat
self._env.filters.update(filters or {})
def _custom_template_names(self, template):
"""Returns a list of custom template names.
template is the name of the original template.
"""
splitted = template.rsplit('/', 1)
name = 'custom_' + splitted[-1]
ret = [name]
if len(splitted) == 2:
ret.append(splitted[0] + '/' + name)
return ret
def render_only(self, template, *args, **kwargs):
"""Renders template template.
The rendered template is returned (no data will be written
or printed).
*args and **kwargs are passed to jinja2 Template's render
method.
"""
names = self._custom_template_names(template)
names.append(template)
tmpl = self._env.select_template(names)
return tmpl.render(*args, **kwargs)
def _render(self, template, out, *args, **kwargs):
"""Renders template template.
out is a file or file-like object to which the rendered
template should be written to.
*args and **kwargs are passed to the render_only method.
"""
text = self.render_only(template, *args, **kwargs)
try:
out.write(text)
except UnicodeEncodeError:
text = text.encode('utf-8')
out.write(text)
def render(self, template, *args, **kwargs):
"""Renders template template.
Writes the rendered template to sys.stdout.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
self._render(template, sys.stdout, *args, **kwargs)
def render_text(self, text, *args, **kwargs):
"""Renders text.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
global TEXT_TEMPLATE
self.render(TEXT_TEMPLATE, text=text, *args, **kwargs)
def render_error(self, template, *args, **kwargs):
"""Renders template template.
Writes the rendered template to sys.stderr.
*args and **kwargs are passed to jinja2 Template's render
method.
"""
self._render(template, sys.stderr, *args, **kwargs)
