async def ds_tiles(metadata=None): datasette = Datasette([], metadata=metadata or {}, memory=True) for db_name, tiles in ( ("world", [[2, 1, 1]]), ("country", [[2, 2, 1], [2, 2, 2]]), ("city1", [[2, 2, 1]]), ("city2", [[2, 3, 3]]), ): db = datasette.add_database(Database(datasette, memory_name=db_name)) # During test runs database tables may exist already if await db.table_exists("tiles"): continue await db.execute_write(CREATE_TILES_TABLE, block=True) await db.execute_write(CREATE_METADATA_TABLE, block=True) for pair in (("name", db_name), ("format", "png")): await db.execute_write( "insert into metadata (name, value) values (?, ?)", pair, block=True, ) for tile in tiles: await db.execute_write( "insert into tiles (zoom_level, tile_column, tile_row, tile_data) values (?, ?, ?, ?)", tile + ["tms:{}:{}".format(db_name, "/".join(map(str, tile)))], block=True, ) await datasette.invoke_startup() return datasette
def test_extensions(): no_extension = render_cell( MARKDOWN_TABLE, column="body_markdown", table="mytable", database="mydatabase", datasette=Datasette([]), ) assert (""" <div style="white-space: normal"><p>First Header | Second Header ------------- | ------------- <a href="https://www.example.com/" rel="nofollow">Content Cell</a> | Content Cell Content Cell | Content Cell</p></div> """.strip() == no_extension) # Now try again with the tables extension with_extension = render_cell( MARKDOWN_TABLE, column="body_markdown", table="mytable", database="mydatabase", datasette=Datasette( [], metadata={ "plugins": { "datasette-render-markdown": { "extensions": ["tables"], "extra_tags": ["table", "thead", "tr", "th", "td", "tbody"], } } }, ), ) assert RENDERED_TABLE == with_extension
def make_app_client( sql_time_limit_ms=None, max_returned_rows=None, cors=False, memory=False, config=None, filename="fixtures.db", is_immutable=False, extra_databases=None, inspect_data=None, static_mounts=None, template_dir=None, metadata=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) if is_immutable: files = [] immutables = [filepath] else: files = [filepath] immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) sqlite3.connect(extra_filepath).executescript(extra_sql) files.append(extra_filepath) os.chdir(os.path.dirname(filepath)) config = config or {} config.update( { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, "sql_time_limit_ms": sql_time_limit_ms or 200, # Default is 3 but this results in "too many open files" # errors when running the full test suite: "num_sql_threads": 1, } ) ds = Datasette( files, immutables=immutables, memory=memory, cors=cors, metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, config=config, inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) client = TestClient(ds.app()) client.ds = ds yield client
def make_app_client( sql_time_limit_ms=None, max_returned_rows=None, cors=False, memory=False, config=None, filename="fixtures.db", is_immutable=False, extra_databases=None, inspect_data=None, static_mounts=None, template_dir=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) if is_immutable: files = [] immutables = [filepath] else: files = [filepath] immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) sqlite3.connect(extra_filepath).executescript(extra_sql) files.append(extra_filepath) os.chdir(os.path.dirname(filepath)) plugins_dir = os.path.join(tmpdir, "plugins") os.mkdir(plugins_dir) open(os.path.join(plugins_dir, "my_plugin.py"), "w").write(PLUGIN1) open(os.path.join(plugins_dir, "my_plugin_2.py"), "w").write(PLUGIN2) config = config or {} config.update( { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, "sql_time_limit_ms": sql_time_limit_ms or 200, } ) ds = Datasette( files, immutables=immutables, memory=memory, cors=cors, metadata=METADATA, plugins_dir=plugins_dir, config=config, inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) client = TestClient(ds.app()) client.ds = ds yield client
def app_client(sql_time_limit_ms=None, max_returned_rows=None, config=None): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, 'fixtures.db') conn = sqlite3.connect(filepath) conn.executescript(TABLES) os.chdir(os.path.dirname(filepath)) plugins_dir = os.path.join(tmpdir, 'plugins') os.mkdir(plugins_dir) open(os.path.join(plugins_dir, 'my_plugin.py'), 'w').write(PLUGIN1) open(os.path.join(plugins_dir, 'my_plugin_2.py'), 'w').write(PLUGIN2) config = config or {} config.update({ 'default_page_size': 50, 'max_returned_rows': max_returned_rows or 100, 'sql_time_limit_ms': sql_time_limit_ms or 200, }) ds = Datasette( [filepath], metadata=METADATA, plugins_dir=plugins_dir, config=config, ) ds.sqlite_functions.append( ('sleep', 1, lambda n: time.sleep(float(n))), ) client = TestClient(ds.app().test_client) client.ds = ds yield client
def make_app_client( max_returned_rows=None, config=None, is_immutable=False, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, 'dummy_tables.db') populate_file(filepath) if is_immutable: files = [] immutables = [filepath] else: files = [filepath] immutables = [] config = config or {} config.update({ 'default_page_size': 50, 'max_returned_rows': max_returned_rows or 1000, }) ds = Datasette( files, immutables=immutables, config=config, ) client = TestClient(ds.app()) client.ds = ds yield client
async def test_indieauth_done_no_params_error(): datasette = Datasette([], memory=True) app = datasette.app() async with httpx.AsyncClient(app=app) as client: response = await client.get("http://localhost/-/indieauth/done") assert response.status_code == 400 assert "Invalid state" in response.text
def app_client( sql_time_limit_ms=None, max_returned_rows=None, cors=False, config=None, filename="fixtures.db", ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) conn = sqlite3.connect(filepath) conn.executescript(TABLES) os.chdir(os.path.dirname(filepath)) plugins_dir = os.path.join(tmpdir, "plugins") os.mkdir(plugins_dir) open(os.path.join(plugins_dir, "my_plugin.py"), "w").write(PLUGIN1) open(os.path.join(plugins_dir, "my_plugin_2.py"), "w").write(PLUGIN2) config = config or {} config.update( { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, "sql_time_limit_ms": sql_time_limit_ms or 200, } ) ds = Datasette( [filepath], cors=cors, metadata=METADATA, plugins_dir=plugins_dir, config=config, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) client = TestClient(ds.app().test_client) client.ds = ds yield client
def test_metadata_yaml(tmp_path_factory, filename): config_dir = tmp_path_factory.mktemp("yaml-config-dir") (config_dir / filename).write_text("title: Title from metadata", "utf-8") ds = Datasette([], config_dir=config_dir) client = _TestClient(ds.app()) client.ds = ds response = client.get("/-/metadata.json") assert 200 == response.status assert {"title": "Title from metadata"} == response.json
async def test_hidden_sqlite_stat1_table(): ds = Datasette() db = ds.add_memory_database("db") await db.execute_write("create table normal (id integer primary key, name text)") await db.execute_write("create index idx on normal (name)") await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] assert tables == [("normal", False), ("sqlite_stat1", True)]
async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") assert db_name in response.json().keys() path = response.json()[db_name]["path"] # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200
async def test_detect_mtiles_databases(i, create_table, should_be_mtiles): datasette = Datasette([]) name = "db_{}".format(i) db = datasette.add_database(Database(datasette, memory_name=name)) if create_table: await db.execute_write(create_table, block=True) result = await detect_mtiles_databases(datasette) expected = [name] if should_be_mtiles else [] assert result == expected
def make_app_client(database, metadata=None): ds = Datasette([database], immutables=[], metadata=metadata, template_dir=str( pathlib.Path(datasette_cldf.__file__).parent / 'templates')) client = Client(ds.app()) client.ds = ds return client
async def test_num_sql_threads_zero(): ds = Datasette([], memory=True, settings={"num_sql_threads": 0}) db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) await db.execute_write("create table t(id integer primary key)") await db.execute_write("insert into t (id) values (1)") response = await ds.client.get("/-/threads.json") assert response.json() == {"num_threads": 0, "threads": []} response2 = await ds.client.get( "/test_num_sql_threads_zero/t.json?_shape=array") assert response2.json() == [{"id": 1}]
async def test_render_template_tag_with_extensions(tmpdir): (tmpdir / "template.html").write_text( '{{ render_markdown("""' + MARKDOWN_TABLE + '""", extensions=["tables"], extra_tags=["table", "thead", "tr", "th", "td", "tbody"]) }}', "utf-8", ) datasette = Datasette([], template_dir=str(tmpdir)) datasette.app() # Configures Jinja rendered = await datasette.render_template(["template.html"]) assert RENDERED_TABLE == rendered
async def test_facet_size(): ds = Datasette([], memory=True, settings={"max_returned_rows": 50}) db = ds.add_database(Database(ds, memory_name="test_facet_size")) await db.execute_write( "create table neighbourhoods(city text, neighbourhood text)", block=True ) for i in range(1, 51): for j in range(1, 4): await db.execute_write( "insert into neighbourhoods (city, neighbourhood) values (?, ?)", ["City {}".format(i), "Neighbourhood {}".format(j)], block=True, ) response = await ds.client.get("/test_facet_size/neighbourhoods.json") data = response.json() assert data["suggested_facets"] == [ { "name": "neighbourhood", "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet=neighbourhood", } ] # Bump up _facet_size= to suggest city too response2 = await ds.client.get( "/test_facet_size/neighbourhoods.json?_facet_size=50" ) data2 = response2.json() assert sorted(data2["suggested_facets"], key=lambda f: f["name"]) == [ { "name": "city", "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city", }, { "name": "neighbourhood", "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=neighbourhood", }, ] # Facet by city should return expected number of results response3 = await ds.client.get( "/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city" ) data3 = response3.json() assert len(data3["facet_results"]["city"]["results"]) == 50 # Reduce max_returned_rows and check that it's respected ds._settings["max_returned_rows"] = 20 response4 = await ds.client.get( "/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city" ) data4 = response4.json() assert len(data4["facet_results"]["city"]["results"]) == 20 # Test _facet_size=max response5 = await ds.client.get( "/test_facet_size/neighbourhoods.json?_facet_size=max&_facet=city" ) data5 = response5.json() assert len(data5["facet_results"]["city"]["results"]) == 20
async def test_schema_caching(tmp_path_factory, db_path, template, expected): template_dir = tmp_path_factory.mktemp("templates") pages_dir = template_dir / "pages" pages_dir.mkdir() (pages_dir / "about.html").write_text(template) ds = Datasette([db_path], template_dir=template_dir) async with httpx.AsyncClient(app=ds.app()) as client: response = await client.get("http://localhost/about") assert response.status_code == 200 assert response.text.strip() == expected
async def test_schema_caching(mock_schema_for_database, tmp_path_factory): mock_schema_for_database.side_effect = schema_for_database db_directory = tmp_path_factory.mktemp("dbs") db_path = db_directory / "schema.db" db = sqlite_utils.Database(db_path) build_database(db) # Previous tests will have populated the cache _schema_cache.clear() assert len(_schema_cache) == 0 # The first hit should call schema_for_database assert not mock_schema_for_database.called ds = Datasette([db_path]) async with httpx.AsyncClient(app=ds.app()) as client: response = await client.get("http://localhost/graphql/schema.graphql") assert response.status_code == 200 assert "view_on_table_with_pkSort" in response.text assert mock_schema_for_database.called assert len(_schema_cache) == 1 mock_schema_for_database.reset_mock() # The secod hit should NOT call it assert not mock_schema_for_database.called async with httpx.AsyncClient(app=ds.app()) as client: response = await client.get("http://localhost/graphql/schema.graphql") assert response.status_code == 200 assert "view_on_table_with_pkSort" in response.text assert "new_table" not in response.text assert not mock_schema_for_database.called current_keys = set(_schema_cache.keys()) assert len(current_keys) == 1 # We change the schema and it should be called again db["new_table"].insert({"new_column": 1}) async with httpx.AsyncClient(app=ds.app()) as client: response = await client.get("http://localhost/graphql/schema.graphql") assert response.status_code == 200 assert "view_on_table_with_pkSort" in response.text assert "new_table" in response.text assert mock_schema_for_database.called assert len(_schema_cache) == 1 assert set(_schema_cache.keys()) != current_keys
def config_dir_client(tmp_path_factory): config_dir = tmp_path_factory.mktemp("config-dir") plugins_dir = config_dir / "plugins" plugins_dir.mkdir() (plugins_dir / "hooray.py").write_text(PLUGIN, "utf-8") templates_dir = config_dir / "templates" templates_dir.mkdir() (templates_dir / "row.html").write_text( "Show row here. Plugin says {{ from_plugin }}", "utf-8") static_dir = config_dir / "static" static_dir.mkdir() (static_dir / "hello.css").write_text(CSS, "utf-8") (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") (config_dir / "config.json").write_text(json.dumps(CONFIG), "utf-8") for dbname in ("demo.db", "immutable.db"): db = sqlite3.connect(str(config_dir / dbname)) db.executescript(""" CREATE TABLE cities ( id integer primary key, name text ); INSERT INTO cities (id, name) VALUES (1, 'San Francisco') ; """) # Mark "immutable.db" as immutable (config_dir / "inspect-data.json").write_text( json.dumps({ "immutable": { "hash": "hash", "size": 8192, "file": "immutable.db", "tables": { "cities": { "count": 1 } }, } }), "utf-8", ) ds = Datasette([], config_dir=config_dir) client = _TestClient(ds.app()) client.ds = ds yield client
async def test_array_facet_handle_duplicate_tags(): ds = Datasette([], memory=True) db = ds.add_database(Database(ds, memory_name="test_array_facet")) await db.execute_write("create table otters(name text, tags text)") for name, tags in ( ("Charles", ["friendly", "cunning", "friendly"]), ("Shaun", ["cunning", "empathetic", "friendly"]), ("Tracy", ["empathetic", "eager"]), ): await db.execute_write( "insert into otters (name, tags) values (?, ?)", [name, json.dumps(tags)] ) response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags") assert response.json()["facet_results"]["tags"] == { "name": "tags", "type": "array", "results": [ { "value": "cunning", "label": "cunning", "count": 2, "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=cunning", "selected": False, }, { "value": "empathetic", "label": "empathetic", "count": 2, "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=empathetic", "selected": False, }, { "value": "friendly", "label": "friendly", "count": 2, "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=friendly", "selected": False, }, { "value": "eager", "label": "eager", "count": 1, "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=eager", "selected": False, }, ], "hideable": True, "toggle_url": "/test_array_facet/otters.json", "truncated": False, }
def app_client(max_returned_rows=None): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, 'test_tables.h5') populate_file(filepath) ds = Datasette( [filepath], config={ 'default_page_size': 50, 'max_returned_rows': max_returned_rows or 1000, } ) client = ds.app().test_client client.ds = ds yield client
async def test_format_of_binary_links(size, title, length_bytes): ds = Datasette() db_name = "binary-links-{}".format(size) db = ds.add_memory_database(db_name) sql = "select zeroblob({}) as blob".format(size) await db.execute_write("create table blobs as {}".format(sql)) response = await ds.client.get("/{}/blobs".format(db_name)) assert response.status_code == 200 expected = "{}><Binary: {} bytes></a>".format(title, length_bytes) assert expected in response.text # And test with arbitrary SQL query too sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text
def app_client(): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, 'test_tables.db') conn = sqlite3.connect(filepath) conn.executescript(TABLES) os.chdir(os.path.dirname(filepath)) ds = Datasette( [filepath], page_size=50, max_returned_rows=100, sql_time_limit_ms=20, ) ds.sqlite_functions.append( ('sleep', 1, lambda n: time.sleep(float(n))), ) yield ds.app().test_client
async def test_dashboard_list_permissions(datasette_db, datasette_metadata, metadata, authenticated, expected_status): datasette = Datasette([str(datasette_db)], metadata={ **datasette_metadata, **metadata }) cookies = {} if authenticated: cookies["ds_actor"] = datasette.sign({"a": {"id": "user"}}, "actor") response = await datasette.client.get("/-/dashboards", cookies=cookies) assert response.status_code == expected_status
async def test_render_template_tag(tmpdir): (tmpdir / "template.html").write_text( """ Demo: {{ render_markdown("* one") }} Done. """.strip(), "utf-8", ) datasette = Datasette([], template_dir=str(tmpdir)) datasette.app() # Configures Jinja rendered = await datasette.render_template(["template.html"]) assert ( 'Demo:\n <div style="white-space: normal"><ul>\n<li>one</li>\n</ul></div>\n Done.' == rendered)
async def test_import_table_multiple_databases(tmpdir): db_path1 = str(tmpdir / "test.db") db_path2 = str(tmpdir / "test2.db") datasette = Datasette([db_path1, db_path2]) cookies = {"ds_actor": datasette.sign({"a": {"id": "root"}}, "actor")} async with httpx.AsyncClient(app=datasette.app()) as client: response = await client.get("http://localhost/-/import-table", cookies=cookies) assert response.status_code == 200 assert "<option>test</option>" in response.text assert "<option>test2</option>" in response.text response2 = await client.get( "http://localhost/-/import-table?database=test2", cookies=cookies) assert response2.status_code == 200 assert '<option selected="selected">test2</option>' in response2.text
async def test_json_array_with_blanks_and_nulls(): ds = Datasette([], memory=True) db = ds.add_database(Database(ds, memory_name="test_json_array")) await db.execute_write("create table foo(json_column text)") for value in ('["a", "b", "c"]', '["a", "b"]', "", None): await db.execute_write("insert into foo (json_column) values (?)", [value]) response = await ds.client.get("/test_json_array/foo.json") data = response.json() assert data["suggested_facets"] == [ { "name": "json_column", "type": "array", "toggle_url": "http://localhost/test_json_array/foo.json?_facet_array=json_column", } ]
async def test_non_matching_authorization_endpoint(httpx_mock): # See https://github.com/simonw/datasette-indieauth/issues/22 httpx_mock.add_response( url="https://simonwillison.net", data= b'<link rel="authorization_endpoint" href="https://indieauth.simonwillison.net/auth">', ) httpx_mock.add_response( url="https://indieauth.simonwillison.net/auth", method="POST", data="me=https%3A%2F%2Fsimonwillison.net%2Fme".encode("utf-8"), ) httpx_mock.add_response( url="https://simonwillison.net/me", data=b'<link rel="authorization_endpoint" href="https://example.com">', ) datasette = Datasette([], memory=True) app = datasette.app() async with httpx.AsyncClient(app=app) as client: csrftoken = await _get_csrftoken(client) # Submit the form post_response = await client.post( "http://localhost/-/indieauth", data={ "csrftoken": csrftoken, "me": "https://simonwillison.net/" }, cookies={"ds_csrftoken": csrftoken}, allow_redirects=False, ) ds_indieauth = post_response.cookies["ds_indieauth"] state = dict( urllib.parse.parse_qsl(post_response.headers["location"].split( "?", 1)[1]))["state"] # ... after redirecting back again response = await client.get( "http://localhost/-/indieauth/done", params={ "state": state, "code": "123", }, cookies={"ds_indieauth": ds_indieauth}, allow_redirects=False, ) # This should be an error because the authorization_endpoint did not match assert ( ""me" value resolves to a different authorization_endpoint" in response.text)
def test_not_emails(): for value in [ "foobar", # just a string "root@localhost", # domain part must include a . "foo@[email protected]", # more than one @ sign ]: assert None is render_cell(value, None, None, None, Datasette([]))
def make_app_client( sql_time_limit_ms=None, max_returned_rows=None, cors=False, memory=False, settings=None, filename="fixtures.db", is_immutable=False, extra_databases=None, inspect_data=None, static_mounts=None, template_dir=None, metadata=None, crossdb=False, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) if is_immutable: files = [] immutables = [filepath] else: files = [filepath] immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) sqlite3.connect(extra_filepath).executescript(extra_sql) # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) settings = settings or {} for key, value in { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, "sql_time_limit_ms": sql_time_limit_ms or 200, # Default is 3 but this results in "too many open files" # errors when running the full test suite: "num_sql_threads": 1, }.items(): if key not in settings: settings[key] = value ds = Datasette( files, immutables=immutables, memory=memory, cors=cors, metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, settings=settings, inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, crossdb=crossdb, ) yield TestClient(ds)
def app_client(): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, 'test_tables.db') conn = sqlite3.connect(filepath) conn.executescript(TABLES) os.chdir(os.path.dirname(filepath)) ds = Datasette( [filepath], page_size=50, max_returned_rows=100, sql_time_limit_ms=20, ) ds.sqlite_functions.append( ('sleep', 1, lambda n: time.sleep(float(n))), ) yield ds.app().test_client