async def get(self, request):
as_format = request.url_vars["format"]
await self.ds.ensure_permissions(request.actor, ["view-instance"])
if self.needs_request:
data = self.data_callback(request)
else:
data = self.data_callback()
if as_format:
headers = {}
if self.ds.cors:
add_cors_headers(headers)
return Response(
json.dumps(data),
content_type="application/json; charset=utf-8",
headers=headers,
)
else:
return await self.render(
["show_json.html"],
request=request,
context={
"filename": self.filename,
"data_json": json.dumps(data, indent=4),
},
)
async def view_get(self, request, database, hash, correct_hash_present,
**kwargs):
await self.check_permissions(
request,
[
("view-database-download", database),
("view-database", database),
"view-instance",
],
)
if database not in self.ds.databases:
raise DatasetteError("Invalid database", status=404)
db = self.ds.databases[database]
if db.is_memory:
raise DatasetteError("Cannot download in-memory databases",
status=404)
if not self.ds.setting("allow_download") or db.is_mutable:
raise Forbidden("Database download is forbidden")
if not db.path:
raise DatasetteError("Cannot download database", status=404)
filepath = db.path
headers = {}
if self.ds.cors:
add_cors_headers(headers)
headers["Transfer-Encoding"] = "chunked"
return AsgiFileDownload(
filepath,
filename=os.path.basename(filepath),
content_type="application/octet-stream",
headers=headers,
)
def redirect(self, request, path, forward_querystring=True, remove_args=None):
if request.query_string and "?" not in path and forward_querystring:
path = f"{path}?{request.query_string}"
if remove_args:
path = path_with_removed_args(request, remove_args, path=path)
r = Response.redirect(path)
r.headers["Link"] = f"<{path}>; rel=preload"
if self.ds.cors:
add_cors_headers(r.headers)
return r
def set_response_headers(self, response, ttl):
# Set far-future cache expiry
if self.ds.cache_headers and response.status == 200:
ttl = int(ttl)
if ttl == 0:
ttl_header = "no-cache"
else:
ttl_header = f"max-age={ttl}"
response.headers["Cache-Control"] = ttl_header
response.headers["Referrer-Policy"] = "no-referrer"
if self.ds.cors:
add_cors_headers(response.headers)
return response
async def get(self, request):
database = tilde_decode(request.url_vars["database"])
await self.ds.ensure_permissions(
request.actor,
[
("view-database-download", database),
("view-database", database),
"view-instance",
],
)
try:
db = self.ds.get_database(route=database)
except KeyError:
raise DatasetteError("Invalid database", status=404)
if db.is_memory:
raise DatasetteError("Cannot download in-memory databases",
status=404)
if not self.ds.setting("allow_download") or db.is_mutable:
raise Forbidden("Database download is forbidden")
if not db.path:
raise DatasetteError("Cannot download database", status=404)
filepath = db.path
headers = {}
if self.ds.cors:
add_cors_headers(headers)
if db.hash:
etag = '"{}"'.format(db.hash)
headers["Etag"] = etag
# Has user seen this already?
if_none_match = request.headers.get("if-none-match")
if if_none_match and if_none_match == etag:
return Response("", status=304)
headers["Transfer-Encoding"] = "chunked"
return AsgiFileDownload(
filepath,
filename=os.path.basename(filepath),
content_type="application/octet-stream",
headers=headers,
)
async def as_csv(self, request, database):
kwargs = {}
stream = request.args.get("_stream")
# Do not calculate facets or counts:
extra_parameters = [
"{}=1".format(key) for key in ("_nofacet", "_nocount")
if not request.args.get(key)
]
if extra_parameters:
# Replace request object with a new one with modified scope
if not request.query_string:
new_query_string = "&".join(extra_parameters)
else:
new_query_string = (request.query_string + "&" +
"&".join(extra_parameters))
new_scope = dict(request.scope,
query_string=new_query_string.encode("latin-1"))
receive = request.receive
request = Request(new_scope, receive)
if stream:
# Some quick soundness checks
if not self.ds.setting("allow_csv_stream"):
raise BadRequest("CSV streaming is disabled")
if request.args.get("_next"):
raise BadRequest("_next not allowed for CSV streaming")
kwargs["_size"] = "max"
# Fetch the first page
try:
response_or_template_contexts = await self.data(request)
if isinstance(response_or_template_contexts, Response):
return response_or_template_contexts
elif len(response_or_template_contexts) == 4:
data, _, _, _ = response_or_template_contexts
else:
data, _, _ = response_or_template_contexts
except (sqlite3.OperationalError, InvalidSql) as e:
raise DatasetteError(str(e), title="Invalid SQL", status=400)
except sqlite3.OperationalError as e:
raise DatasetteError(str(e))
except DatasetteError:
raise
# Convert rows and columns to CSV
headings = data["columns"]
# if there are expanded_columns we need to add additional headings
expanded_columns = set(data.get("expanded_columns") or [])
if expanded_columns:
headings = []
for column in data["columns"]:
headings.append(column)
if column in expanded_columns:
headings.append(f"{column}_label")
content_type = "text/plain; charset=utf-8"
preamble = ""
postamble = ""
trace = request.args.get("_trace")
if trace:
content_type = "text/html; charset=utf-8"
preamble = ("<html><head><title>CSV debug</title></head>"
'<body><textarea style="width: 90%; height: 70vh">')
postamble = "</textarea></body></html>"
async def stream_fn(r):
nonlocal data, trace
limited_writer = LimitedWriter(r, self.ds.setting("max_csv_mb"))
if trace:
await limited_writer.write(preamble)
writer = csv.writer(EscapeHtmlWriter(limited_writer))
else:
writer = csv.writer(limited_writer)
first = True
next = None
while first or (next and stream):
try:
kwargs = {}
if next:
kwargs["_next"] = next
if not first:
data, _, _ = await self.data(request, **kwargs)
if first:
if request.args.get("_header") != "off":
await writer.writerow(headings)
first = False
next = data.get("next")
for row in data["rows"]:
if any(isinstance(r, bytes) for r in row):
new_row = []
for column, cell in zip(headings, row):
if isinstance(cell, bytes):
# If this is a table page, use .urls.row_blob()
if data.get("table"):
pks = data.get("primary_keys") or []
cell = self.ds.absolute_url(
request,
self.ds.urls.row_blob(
database,
data["table"],
path_from_row_pks(
row, pks, not pks),
column,
),
)
else:
# Otherwise generate URL for this query
url = self.ds.absolute_url(
request,
path_with_format(
request=request,
format="blob",
extra_qs={
"_blob_column":
column,
"_blob_hash":
hashlib.sha256(
cell).hexdigest(),
},
replace_format="csv",
),
)
cell = url.replace("&_nocount=1",
"").replace(
"&_nofacet=1",
"")
new_row.append(cell)
row = new_row
if not expanded_columns:
# Simple path
await writer.writerow(row)
else:
# Look for {"value": "label": } dicts and expand
new_row = []
for heading, cell in zip(data["columns"], row):
if heading in expanded_columns:
if cell is None:
new_row.extend(("", ""))
else:
assert isinstance(cell, dict)
new_row.append(cell["value"])
new_row.append(cell["label"])
else:
new_row.append(cell)
await writer.writerow(new_row)
except Exception as e:
sys.stderr.write("Caught this error: {}\n".format(e))
sys.stderr.flush()
await r.write(str(e))
return
await limited_writer.write(postamble)
headers = {}
if self.ds.cors:
add_cors_headers(headers)
if request.args.get("_dl", None):
if not trace:
content_type = "text/csv; charset=utf-8"
disposition = 'attachment; filename="{}.csv"'.format(
request.url_vars.get("table", database))
headers["content-disposition"] = disposition
return AsgiStream(stream_fn,
headers=headers,
content_type=content_type)
async def options(self, request, *args, **kwargs):
r = Response.text("ok")
if self.ds.cors:
add_cors_headers(r.headers)
return r
async def get(self, request, as_format):
await self.check_permission(request, "view-instance")
databases = []
for name, db in self.ds.databases.items():
visible, database_private = await check_visibility(
self.ds,
request.actor,
"view-database",
name,
)
if not visible:
continue
table_names = await db.table_names()
hidden_table_names = set(await db.hidden_table_names())
views = []
for view_name in await db.view_names():
visible, private = await check_visibility(
self.ds,
request.actor,
"view-table",
(name, view_name),
)
if visible:
views.append({"name": view_name, "private": private})
# Perform counts only for immutable or DBS with <= COUNT_TABLE_LIMIT tables
table_counts = {}
if not db.is_mutable or db.size < COUNT_DB_SIZE_LIMIT:
table_counts = await db.table_counts(10)
# If any of these are None it means at least one timed out - ignore them all
if any(v is None for v in table_counts.values()):
table_counts = {}
tables = {}
for table in table_names:
visible, private = await check_visibility(
self.ds,
request.actor,
"view-table",
(name, table),
)
if not visible:
continue
table_columns = await db.table_columns(table)
tables[table] = {
"name": table,
"columns": table_columns,
"primary_keys": await db.primary_keys(table),
"count": table_counts.get(table),
"hidden": table in hidden_table_names,
"fts_table": await db.fts_table(table),
"num_relationships_for_sorting": 0,
"private": private,
}
if request.args.get(
"_sort") == "relationships" or not table_counts:
# We will be sorting by number of relationships, so populate that field
all_foreign_keys = await db.get_all_foreign_keys()
for table, foreign_keys in all_foreign_keys.items():
if table in tables.keys():
count = len(foreign_keys["incoming"] +
foreign_keys["outgoing"])
tables[table]["num_relationships_for_sorting"] = count
hidden_tables = [t for t in tables.values() if t["hidden"]]
visible_tables = [t for t in tables.values() if not t["hidden"]]
tables_and_views_truncated = list(
sorted(
(t for t in tables.values() if t not in hidden_tables),
key=lambda t: (
t["num_relationships_for_sorting"],
t["count"] or 0,
t["name"],
),
reverse=True,
)[:TRUNCATE_AT])
# Only add views if this is less than TRUNCATE_AT
if len(tables_and_views_truncated) < TRUNCATE_AT:
num_views_to_add = TRUNCATE_AT - len(
tables_and_views_truncated)
for view in views[:num_views_to_add]:
tables_and_views_truncated.append(view)
databases.append({
"name":
name,
"hash":
db.hash,
"color":
db.hash[:6] if db.hash else hashlib.md5(
name.encode("utf8")).hexdigest()[:6],
"path":
self.ds.urls.database(name),
"tables_and_views_truncated":
tables_and_views_truncated,
"tables_and_views_more":
(len(visible_tables) + len(views)) > TRUNCATE_AT,
"tables_count":
len(visible_tables),
"table_rows_sum":
sum((t["count"] or 0) for t in visible_tables),
"show_table_row_counts":
bool(table_counts),
"hidden_table_rows_sum":
sum(t["count"] for t in hidden_tables
if t["count"] is not None),
"hidden_tables_count":
len(hidden_tables),
"views_count":
len(views),
"private":
database_private,
})
if as_format:
headers = {}
if self.ds.cors:
add_cors_headers(headers)
return Response(
json.dumps({db["name"]: db
for db in databases},
cls=CustomJSONEncoder),
content_type="application/json; charset=utf-8",
headers=headers,
)
else:
return await self.render(
["index.html"],
request=request,
context={
"databases":
databases,
"metadata":
self.ds.metadata(),
"datasette_version":
__version__,
"private":
not await self.ds.permission_allowed(
None, "view-instance", default=True),
},
)