async def test(pgdata_path):
async with tb.start_edgedb_server(
backend_dsn=
f'postgres:///?user=postgres&host={pgdata_path}',
reset_auth=True,
runstate_dir=None
if devmode.is_in_dev_mode() else pgdata_path,
) as sd:
con = await sd.connect()
try:
val = await con.query_single('SELECT 123')
self.assertEqual(int(val), 123)
# stop the postgres
await cluster.stop()
with self.assertRaisesRegex(
errors.BackendUnavailableError,
'Postgres is not available',
):
await con.query_single('SELECT 123+456')
# bring postgres back
await cluster.start()
# give the EdgeDB server some time to recover
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
try:
val = await con.query_single('SELECT 123+456')
break
except errors.BackendUnavailableError:
pass
self.assertEqual(int(val), 579)
finally:
await con.aclose()
async def test_server_config_query_timeout(self):
async with tb.start_edgedb_server(
http_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
conn = await sd.connect()
await conn.execute('''
configure session set
query_execution_timeout :=
<duration>'1 second'
''')
for _ in range(2):
with self.assertRaisesRegex(
edgedb.QueryError,
'canceling statement due to statement timeout'):
await conn.execute('''
select sys::_sleep(4)
''')
self.assertEqual(await conn.query_single('select 42'), 42)
await conn.aclose()
data = sd.fetch_metrics()
self.assertNotIn(
'\nedgedb_server_backend_connections_aborted_total', data)
async def test_server_ops_temp_dir(self):
# Test that "edgedb-server" works as expected with the
# following arguments:
#
# * "--port=auto"
# * "--temp-dir"
# * "--auto-shutdown"
# * "--emit-server-status"
async with tb.start_edgedb_server(auto_shutdown=True, ) as sd:
con1 = await sd.connect()
self.assertEqual(await con1.query_one('SELECT 1'), 1)
con2 = await sd.connect()
self.assertEqual(await con2.query_one('SELECT 1'), 1)
await con1.aclose()
self.assertEqual(await con2.query_one('SELECT 42'), 42)
await con2.aclose()
with self.assertRaises(
(ConnectionError, edgedb.ClientConnectionError)):
# Since both con1 and con2 are now disconnected and
# the cluster was started with an "--auto-shutdown"
# option, we expect this connection to be rejected
# and the cluster to be shutdown soon.
await edgedb.async_connect(
user='******',
host=sd.host,
port=sd.port,
wait_until_available=0,
)
async def test_proto_gh3170_connection_lost_error(self):
async with tb.start_edgedb_server(
security=srv_args.ServerSecurityMode.InsecureDevMode,
) as sd:
self.assertNotIn(
'edgedb_server_background_errors_total'
'{source="release_pgcon"}',
sd.fetch_metrics(),
)
con = await sd.connect_test_protocol()
try:
await con.send(
protocol.ExecuteScript(
headers=[],
script='START TRANSACTION'
)
)
await con.recv_match(
protocol.CommandComplete,
status='START TRANSACTION'
)
await con.recv_match(
protocol.ReadyForCommand,
transaction_state=protocol.TransactionState.IN_TRANSACTION,
)
await con.aclose()
self.assertNotIn(
'edgedb_server_background_errors_total'
'{source="release_pgcon"}',
sd.fetch_metrics(),
)
except Exception:
await con.aclose()
raise
async def test_server_config_idle_connection_01(self):
async with tb.start_edgedb_server(
http_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
active_cons = []
idle_cons = []
for i in range(20):
if i % 2:
active_cons.append(await sd.connect())
else:
idle_cons.append(await sd.connect())
# Set the timeout to 5 seconds.
await idle_cons[0].execute('''
configure system set session_idle_timeout := <duration>'5s'
''')
for _ in range(5):
random.shuffle(active_cons)
await asyncio.gather(*(con.query('SELECT 1')
for con in active_cons))
await asyncio.sleep(3)
metrics = sd.fetch_metrics()
await asyncio.gather(*(con.aclose() for con in active_cons))
self.assertIn(
f'\nedgedb_server_client_connections_idle_total ' +
f'{float(len(idle_cons))}\n', metrics)
async def test_server_proto_configure_listen_addresses(self):
con1 = None
con2 = None
async with tb.start_edgedb_server(auto_shutdown=True) as sd:
try:
con1 = await sd.connect()
await con1.execute("""
CONFIGURE INSTANCE SET listen_addresses := {
'127.0.0.2',
};
""")
con2 = await sd.connect(host="127.0.0.2")
self.assertEqual(await con1.query_single("SELECT 1"), 1)
self.assertEqual(await con2.query_single("SELECT 2"), 2)
finally:
closings = []
if con1 is not None:
closings.append(con1.aclose())
if con2 is not None:
closings.append(con2.aclose())
await asyncio.gather(*closings)
async def _test_server_ops_ignore_other_tenants(self, td, user):
async with tb.start_edgedb_server(
backend_dsn=f'postgres:///?user={user}&host={td}',
runstate_dir=None if devmode.is_in_dev_mode() else td,
reset_auth=True,
) as sd:
con = await sd.connect()
await con.aclose()
async with tb.start_edgedb_server(
backend_dsn=f'postgres:///?user=postgres&host={td}',
runstate_dir=None if devmode.is_in_dev_mode() else td,
reset_auth=True,
ignore_other_tenants=True,
env={'EDGEDB_TEST_CATALOG_VERSION': '3022_01_07_00_00'},
) as sd:
con = await sd.connect()
await con.aclose()
async def test_server_ops_generates_cert_to_specified_file(self):
cert_fd, cert_file = tempfile.mkstemp()
os.close(cert_fd)
os.unlink(cert_file)
key_fd, key_file = tempfile.mkstemp()
os.close(key_fd)
os.unlink(key_file)
try:
async with tb.start_edgedb_server(
tls_cert_file=cert_file,
tls_key_file=key_file,
) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
key_file_path = pathlib.Path(key_file)
cert_file_path = pathlib.Path(cert_file)
self.assertTrue(key_file_path.exists())
self.assertTrue(cert_file_path.exists())
self.assertGreater(key_file_path.stat().st_size, 0)
self.assertGreater(cert_file_path.stat().st_size, 0)
# Check that the server works with the generated cert/key
async with tb.start_edgedb_server(
tls_cert_file=cert_file,
tls_key_file=key_file,
tls_cert_mode=args.ServerTlsCertMode.RequireFile,
) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
finally:
os.unlink(key_file)
os.unlink(cert_file)
async def test_server_ops_cleartext_http_allowed(self):
async with tb.start_edgedb_server(
auto_shutdown=True,
allow_insecure_http_clients=True,
) as sd:
con = http.client.HTTPConnection(sd.host, sd.port)
con.connect()
try:
con.request('GET', f'http://{sd.host}:{sd.port}/blah404')
resp = con.getresponse()
self.assertEqual(resp.status, 404)
finally:
con.close()
tls_context = ssl.create_default_context(
ssl.Purpose.SERVER_AUTH,
cafile=sd.tls_cert_file,
)
tls_context.check_hostname = False
con = http.client.HTTPSConnection(sd.host,
sd.port,
context=tls_context)
con.connect()
try:
con.request('GET', f'http://{sd.host}:{sd.port}/blah404')
resp = con.getresponse()
self.assertEqual(resp.status, 404)
resp_headers = {
k.lower(): v.lower()
for k, v in resp.getheaders()
}
self.assertIn('strict-transport-security', resp_headers)
# When --allow-insecure-http-clients is passed, we set
# max-age to 0, to let browsers know that it's safe
# for the user to open http://
self.assertEqual(resp_headers['strict-transport-security'],
'max-age=0')
finally:
con.close()
# Connect to let it autoshutdown; also test that
# --allow-insecure-http-clients doesn't break binary
# connections.
con = await edb_protocol.new_connection(
user='******',
password=sd.password,
host=sd.host,
port=sd.port,
tls_ca_file=sd.tls_cert_file,
)
try:
await self._test_connection(con)
finally:
await con.aclose()
async def test_server_proto_configure_listen_addresses(self):
con1 = con2 = con3 = con4 = con5 = None
async with tb.start_edgedb_server() as sd:
try:
with self.assertRaises(
edgedb.ClientConnectionFailedTemporarilyError):
await sd.connect(host="127.0.0.2",
timeout=1,
wait_until_available=1)
con1 = await sd.connect()
await con1.execute("""
CONFIGURE INSTANCE SET listen_addresses := {
'127.0.0.2',
};
""")
con2 = await sd.connect(host="127.0.0.2")
self.assertEqual(await con1.query_single("SELECT 1"), 1)
self.assertEqual(await con2.query_single("SELECT 2"), 2)
with self.assertRaises(
edgedb.ClientConnectionFailedTemporarilyError):
await sd.connect(timeout=1, wait_until_available=1)
await con1.execute("""
CONFIGURE INSTANCE SET listen_addresses := {
'127.0.0.1', '127.0.0.2',
};
""")
con3 = await sd.connect()
for i, con in enumerate((con1, con2, con3)):
self.assertEqual(await con.query_single(f"SELECT {i}"), i)
await con1.execute("""
CONFIGURE INSTANCE SET listen_addresses := <str>{};
""")
await con1.execute("""
CONFIGURE INSTANCE SET listen_addresses := {
'0.0.0.0',
};
""")
con4 = await sd.connect()
con5 = await sd.connect(host="127.0.0.2")
for i, con in enumerate((con1, con2, con3, con4, con5)):
self.assertEqual(await con.query_single(f"SELECT {i}"), i)
finally:
closings = []
for con in (con1, con2, con3, con4, con5):
if con is not None:
closings.append(con.aclose())
await asyncio.gather(*closings)
async def test_server_ops_bogus_bind_addr_in_mix(self):
async with tb.start_edgedb_server(bind_addrs=(
'host.invalid',
'127.0.0.1',
), ) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
async def test_server_ops_set_pg_max_connections(self):
actual = random.randint(50, 100)
async with tb.start_edgedb_server(
max_allowed_connections=actual, ) as sd:
con = await sd.connect()
try:
max_connections = await con.query_single(
'SELECT cfg::InstanceConfig.__pg_max_connections LIMIT 1'
) # TODO: remove LIMIT 1 after #2402
self.assertEqual(int(max_connections), actual + 2)
finally:
await con.aclose()
async def test_server_ops_bogus_bind_addr_only(self):
with self.assertRaisesRegex(
edbcluster.ClusterError,
"could not create any listen sockets",
):
async with tb.start_edgedb_server(
bind_addrs=('host.invalid', ), ) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
async def test_server_ops_downgrade_to_cleartext(self):
async with tb.start_edgedb_server(
binary_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
con = await sd.connect_test_protocol(
user='******',
tls_security='insecure',
)
try:
await self._test_connection(con)
finally:
await con.aclose()
async def test_server_config_db_config(self):
async with tb.start_edgedb_server(
http_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
con1 = await sd.connect()
con2 = await sd.connect()
await con1.execute('''
configure current database set __internal_sess_testvalue := 0;
''')
await con2.execute('''
configure current database set __internal_sess_testvalue := 5;
''')
# Check that the DB (Backend) was updated.
conf = await con2.query_single('''
SELECT assert_single(cfg::Config.__internal_sess_testvalue)
''')
self.assertEqual(conf, 5)
# The changes should be immediately visible at EdgeQL level
# in concurrent transactions.
conf = await con1.query_single('''
SELECT assert_single(cfg::Config.__internal_sess_testvalue)
''')
self.assertEqual(conf, 5)
# Use `try_until_succeeds` because it might take the server a few
# seconds on slow CI to reload the DB config in the server process.
async for tr in self.try_until_succeeds(ignore=AssertionError):
async with tr:
info = sd.fetch_server_info()
dbconf = info['databases']['edgedb']['config']
self.assertEqual(dbconf.get('__internal_sess_testvalue'),
5)
# Now check that the server state is updated when a configure
# command is in a transaction.
async for tx in con1.retrying_transaction():
async with tx:
await tx.execute('''
configure current database set
__internal_sess_testvalue := 10;
''')
async for tr in self.try_until_succeeds(ignore=AssertionError):
async with tr:
info = sd.fetch_server_info()
dbconf = info['databases']['edgedb']['config']
self.assertEqual(dbconf.get('__internal_sess_testvalue'),
10)
async def test_server_ops_no_cleartext(self):
async with tb.start_edgedb_server(
auto_shutdown=True,
allow_insecure_binary_clients=False,
allow_insecure_http_clients=False,
) as sd:
con = http.client.HTTPConnection(sd.host, sd.port)
con.connect()
try:
con.request('GET', f'http://{sd.host}:{sd.port}/blah404')
resp = con.getresponse()
self.assertEqual(resp.status, 301)
resp_headers = {
k.lower(): v.lower()
for k, v in resp.getheaders()
}
self.assertIn('location', resp_headers)
self.assertTrue(
resp_headers['location'].startswith('https://'))
self.assertIn('strict-transport-security', resp_headers)
# By default we enforce HTTPS via HSTS on all routes.
self.assertEqual(resp_headers['strict-transport-security'],
'max-age=31536000')
finally:
con.close()
con = await edb_protocol.new_connection(
user='******',
password=sd.password,
host=sd.host,
port=sd.port,
use_tls=False,
)
try:
with self.assertRaisesRegex(errors.BinaryProtocolError,
"TLS Required"):
await con.connect()
finally:
await con.aclose()
con = await edb_protocol.new_connection(
user='******',
password=sd.password,
host=sd.host,
port=sd.port,
tls_ca_file=sd.tls_cert_file,
)
try:
await self._test_connection(con)
finally:
await con.aclose()
async def test_server_ops_generates_cert_to_default_location(self):
with tempfile.TemporaryDirectory() as temp_dir:
async with tb.start_edgedb_server(
data_dir=temp_dir,
default_auth_method=args.ServerAuthMethod.Trust,
) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
# Check that the server works with the generated cert/key
async with tb.start_edgedb_server(
data_dir=temp_dir,
tls_cert_mode=args.ServerTlsCertMode.RequireFile,
default_auth_method=args.ServerAuthMethod.Trust,
) as sd:
con = await sd.connect()
try:
await con.query_single("SELECT 1")
finally:
await con.aclose()
async def test_server_ops_bootstrap_script_server(self):
# Test that "edgedb-server" works as expected with the
# following arguments:
#
# * "--bootstrap-command"
async with tb.start_edgedb_server(
bootstrap_command='CREATE SUPERUSER ROLE test_bootstrap2 '
'{ SET password := "******" };') as sd:
con = await sd.connect(user='******', password='******')
try:
self.assertEqual(await con.query_single('SELECT 1'), 1)
finally:
await con.aclose()
async def test_server_only_bootstraps_once(self):
with tempfile.TemporaryDirectory() as temp_dir:
async with tb.start_edgedb_server(
data_dir=temp_dir,
default_auth_method=args.ServerAuthMethod.Scram,
bootstrap_command=
'ALTER ROLE edgedb SET password := "******";') as sd:
con = await sd.connect(password='******')
try:
await con.query_single('SELECT 1')
finally:
await con.aclose()
# The bootstrap command should not be run on subsequent server starts.
async with tb.start_edgedb_server(
data_dir=temp_dir,
default_auth_method=args.ServerAuthMethod.Scram,
bootstrap_command=
'ALTER ROLE edgedb SET password := "******";') as sd:
con = await sd.connect(password='******')
try:
await con.query_single('SELECT 1')
finally:
await con.aclose()
async def test_ha_adaptive(self):
debug = False
env = dict(EDGEDB_SERVER_BACKEND_ADAPTIVE_HA_UNHEALTHY_MIN_TIME="3")
async with stolon_setup(debug=debug) as (consul, pg1, pg2):
async with AdaptiveHAProxy(consul.http_port, debug=debug) as port:
async with tb.start_edgedb_server(
backend_dsn=
(f"postgresql://*****:*****@127.0.0.1:{port}/postgres"),
runstate_dir=str(
pathlib.Path(consul.tmp_dir.name) / "edb"),
enable_backend_adaptive_ha=True,
reset_auth=True,
debug=debug,
env=env,
) as sd:
await self._test_failover(pg1, pg2, sd, debug=debug)
async def test_server_ops_downgrade_to_cleartext(self):
async with tb.start_edgedb_server(
auto_shutdown=True,
allow_insecure_binary_clients=True,
) as sd:
con = await edb_protocol.new_connection(
user='******',
password=sd.password,
host=sd.host,
port=sd.port,
use_tls=False,
)
try:
await self._test_connection(con)
finally:
await con.aclose()
async def test_server_config_idle_connection_02(self):
from edb import protocol
async with tb.start_edgedb_server(
http_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
conn = await sd.connect_test_protocol()
await conn.simple_query('''
configure system set session_idle_timeout := <duration>'10ms'
''')
await asyncio.sleep(1)
await conn.recv_match(
protocol.ErrorResponse,
message='closing the connection due to idling')
async def _fixture(self):
# Prepare the test data
con1 = con2 = None
async with tb.start_edgedb_server(max_allowed_connections=4) as sd:
conn_args = sd.get_connect_args()
try:
con1 = await sd.connect_test_protocol()
con2 = await sd.connect()
await con2.execute(
'CREATE TYPE tclcq { CREATE PROPERTY p -> str }'
)
await con2.execute("INSERT tclcq { p := 'initial' }")
yield con1, con2, conn_args
finally:
for con in [con1, con2]:
if con is not None:
await con.aclose()
async def test(pgdata_path):
async with tb.start_edgedb_server(
auto_shutdown=True,
max_allowed_connections=None,
postgres_dsn=
f'postgres:///?user=postgres&host={pgdata_path}',
reset_auth=True,
runstate_dir=None
if devmode.is_in_dev_mode() else pgdata_path,
) as sd:
con = await sd.connect()
try:
max_connections = await con.query_one(
'SELECT cfg::SystemConfig.__pg_max_connections LIMIT 1'
) # TODO: remove LIMIT 1 after #2402
self.assertEqual(int(max_connections), actual)
finally:
await con.aclose()
async def test_server_ops_temp_dir(self):
# Test that "edgedb-server" works as expected with the
# following arguments:
#
# * "--port=auto"
# * "--temp-dir"
# * "--auto-shutdown"
# * "--echo-runtime-info"
bootstrap_command = (r'CONFIGURE SYSTEM INSERT Auth '
r'{ priority := 0, method := (INSERT Trust) }')
async with tb.start_edgedb_server(bootstrap_command=bootstrap_command,
auto_shutdown=True) as sd:
self.assertTrue(os.path.exists(sd.host))
con1 = await edgedb.async_connect(user='******',
host=sd.host,
port=sd.port)
self.assertEqual(await con1.query_one('SELECT 1'), 1)
con2 = await edgedb.async_connect(user='******',
host=sd.host,
port=sd.port)
self.assertEqual(await con2.query_one('SELECT 1'), 1)
await con1.aclose()
self.assertEqual(await con2.query_one('SELECT 42'), 42)
await con2.aclose()
with self.assertRaises(
(ConnectionError, edgedb.ClientConnectionError)):
# Since both con1 and con2 are now disconnected and
# the cluster was started with an "--auto-shutdown"
# option, we expect this connection to be rejected
# and the cluster to be shutdown soon.
await edgedb.async_connect(
user='******',
host=sd.host,
port=sd.port,
wait_until_available=0,
)
async def test(pgdata_path, tenant):
async with tb.start_edgedb_server(
tenant_id=tenant,
reset_auth=True,
backend_dsn=
f'postgres:///?user=postgres&host={pgdata_path}',
runstate_dir=None
if devmode.is_in_dev_mode() else pgdata_path,
) as sd:
con = await sd.connect()
try:
await con.execute(f'CREATE DATABASE {tenant}')
await con.execute(f'CREATE SUPERUSER ROLE {tenant}')
databases = await con.query('SELECT sys::Database.name')
self.assertEqual(set(databases), {'edgedb', tenant})
roles = await con.query('SELECT sys::Role.name')
self.assertEqual(set(roles), {'edgedb', tenant})
finally:
await con.aclose()
async def test_ha_stolon(self):
debug = False
async with stolon_setup(debug=debug) as (consul, pg1, pg2):
if debug:
print("=" * 80)
print("Stolon is ready")
async with tb.start_edgedb_server(
backend_dsn=(
f"stolon+consul+http://127.0.0.1:{consul.http_port}"
f"/{pg1.cluster_name}"),
runstate_dir=str(
pathlib.Path(consul.tmp_dir.name) / "edb"),
env=dict(
PGUSER="******",
PGPASSWORD="******",
PGDATABASE="postgres",
),
reset_auth=True,
debug=debug,
) as sd:
await self._test_failover(pg1, pg2, sd, debug=debug)
async def test(host):
bootstrap_command = (
r'CONFIGURE SYSTEM INSERT Auth '
r'{ priority := 0, method := (INSERT Trust) }')
async with tb.start_edgedb_server(
auto_shutdown=True,
bootstrap_command=bootstrap_command,
max_allowed_connections=None,
postgres_dsn=f'postgres:///?user=postgres&port={port}&'
f'host={host}',
) as sd:
con = await edgedb.async_connect(user='******',
host=sd.host,
port=sd.port)
try:
max_connections = await con.query_one(
'SELECT cfg::SystemConfig.__pg_max_connections '
'LIMIT 1') # TODO: remove LIMIT 1 after #2402
self.assertEqual(int(max_connections), actual)
finally:
await con.aclose()
async def test_server_config_idle_transaction(self):
from edb import protocol
async with tb.start_edgedb_server(
http_endpoint_security=args.ServerEndpointSecurityMode.
Optional, ) as sd:
conn = await sd.connect_test_protocol()
await conn.simple_query('''
configure session set
session_idle_transaction_timeout :=
<duration>'1 second'
''')
await conn.simple_query('''
start transaction
''')
await conn.simple_query('''
select sys::_sleep(4)
''')
await conn.recv_match(protocol.ErrorResponse,
message='terminating connection due to '
'idle-in-transaction timeout')
with self.assertRaises(edgedb.ClientConnectionClosedError):
await conn.simple_query('''
select 1
''')
data = sd.fetch_metrics()
# Postgres: ERROR_IDLE_IN_TRANSACTION_TIMEOUT=25P03
self.assertIn(
'\nedgedb_server_backend_connections_aborted_total' +
'{pgcode="25P03"} 1.0\n', data)
async def test_server_ops_set_pg_max_connections(self):
bootstrap_command = (r'CONFIGURE SYSTEM INSERT Auth '
r'{ priority := 0, method := (INSERT Trust) }')
actual = random.randint(50, 100)
async with tb.start_edgedb_server(
auto_shutdown=True,
bootstrap_command=bootstrap_command,
max_allowed_connections=actual,
) as sd:
run_dir = sd.server_data['runstate_dir']
port = 0
for sock in pathlib.Path(run_dir).glob('.s.PGSQL.*'):
if sock.is_socket():
port = int(sock.suffix[1:])
break
con = await edgedb.async_connect(user='******',
host=sd.host,
port=sd.port)
self.assertEqual(await con.query_one('SELECT 1'), 1)
try:
conn = await asyncpg.connect(host=run_dir,
port=port,
user='******')
try:
max_connectiosn = await conn.fetchval(
"SELECT setting FROM pg_settings "
"WHERE name = 'max_connections'")
self.assertEqual(int(max_connectiosn), actual)
finally:
await conn.close()
finally:
await con.aclose()
