async def get_local_pg_cluster(
data_dir: pathlib.Path,
*,
runstate_dir: Optional[pathlib.Path] = None,
max_connections: Optional[int] = None,
tenant_id: Optional[str] = None,
log_level: Optional[str] = None,
) -> Cluster:
if log_level is None:
log_level = 'i'
if tenant_id is None:
tenant_id = buildmeta.get_default_tenant_id()
instance_params = None
if max_connections is not None:
instance_params = pgparams.get_default_runtime_params(
max_connections=max_connections,
tenant_id=tenant_id,
).instance_params
cluster = Cluster(
data_dir=data_dir,
runstate_dir=runstate_dir,
instance_params=instance_params,
log_level=log_level,
)
await cluster.lookup_postgres()
return cluster
def get_default_runtime_params(**instance_params: Any,
) -> BackendRuntimeParams:
capabilities = ALL_BACKEND_CAPABILITIES
if not _is_c_utf8_locale_present():
capabilities &= ~BackendCapabilities.C_UTF8_LOCALE
instance_params.setdefault('capabilities', capabilities)
if 'tenant_id' not in instance_params:
instance_params = dict(
tenant_id=buildmeta.get_default_tenant_id(),
**instance_params,
)
if 'version' not in instance_params:
instance_params = dict(
version=buildmeta.get_pg_version(),
**instance_params,
)
return BackendRuntimeParams(
instance_params=BackendInstanceParams(**instance_params), )
async def get_remote_pg_cluster(
dsn: str,
*,
tenant_id: Optional[str] = None,
) -> RemoteCluster:
parsed = urllib.parse.urlparse(dsn)
ha_backend = None
if parsed.scheme not in {'postgresql', 'postgres'}:
ha_backend = ha_base.get_backend(parsed)
if ha_backend is None:
raise ValueError(
'invalid DSN: scheme is expected to be "postgresql", '
'"postgres" or one of the supported HA backend, '
'got {!r}'.format(parsed.scheme))
addr = await ha_backend.get_cluster_consensus()
dsn = 'postgresql://{}:{}'.format(*addr)
addrs, params = pgconnparams.parse_dsn(dsn)
if len(addrs) > 1:
raise ValueError('multiple hosts in Postgres DSN are not supported')
if tenant_id is None:
t_id = buildmeta.get_default_tenant_id()
else:
t_id = tenant_id
rcluster = RemoteCluster(addrs[0], params)
async def _get_cluster_type(
conn: asyncpg.Connection,
) -> Tuple[Type[RemoteCluster], Optional[str]]:
managed_clouds = {
'rds_superuser': RemoteCluster, # Amazon RDS
'cloudsqlsuperuser': RemoteCluster, # GCP Cloud SQL
'azure_pg_admin': RemoteCluster, # Azure Postgres
}
managed_cloud_super = await conn.fetchval(
"""
SELECT
rolname
FROM
pg_roles
WHERE
rolname = any($1::text[])
LIMIT
1
""",
list(managed_clouds),
)
if managed_cloud_super is not None:
return managed_clouds[managed_cloud_super], managed_cloud_super
else:
return RemoteCluster, None
async def _detect_capabilities(
conn: asyncpg.Connection, ) -> pgparams.BackendCapabilities:
caps = pgparams.BackendCapabilities.NONE
try:
cur_cluster_name = await conn.fetchval(f'''
SELECT
setting
FROM
pg_file_settings
WHERE
setting = 'cluster_name'
AND sourcefile = ((
SELECT setting
FROM pg_settings WHERE name = 'data_directory'
) || '/postgresql.auto.conf')
''')
except asyncpg.InsufficientPrivilegeError:
configfile_access = False
else:
try:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name = 'edgedb-test'
""")
except asyncpg.InsufficientPrivilegeError:
configfile_access = False
except asyncpg.InternalServerError as e:
# Stolon keeper symlinks postgresql.auto.conf to /dev/null
# making ALTER SYSTEM fail with InternalServerError,
# see https://github.com/sorintlab/stolon/pull/343
if 'could not fsync file "postgresql.auto.conf"' in e.args[0]:
configfile_access = False
else:
raise
else:
configfile_access = True
if cur_cluster_name:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name =
'{pgcommon.quote_literal(cur_cluster_name)}'
""")
else:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name = DEFAULT
""")
if configfile_access:
caps |= pgparams.BackendCapabilities.CONFIGFILE_ACCESS
tx = conn.transaction()
await tx.start()
rname = str(uuidgen.uuid1mc())
try:
await conn.execute(f'CREATE ROLE "{rname}" WITH SUPERUSER')
except asyncpg.InsufficientPrivilegeError:
can_make_superusers = False
else:
can_make_superusers = True
finally:
await tx.rollback()
if can_make_superusers:
caps |= pgparams.BackendCapabilities.SUPERUSER_ACCESS
coll = await conn.fetchval('''
SELECT collname FROM pg_collation
WHERE lower(replace(collname, '-', '')) = 'c.utf8' LIMIT 1;
''')
if coll is not None:
caps |= pgparams.BackendCapabilities.C_UTF8_LOCALE
roles = await conn.fetchrow('''
SELECT rolcreaterole, rolcreatedb FROM pg_roles
WHERE rolname = (SELECT current_user);
''')
if roles['rolcreaterole']:
caps |= pgparams.BackendCapabilities.CREATE_ROLE
if roles['rolcreatedb']:
caps |= pgparams.BackendCapabilities.CREATE_DATABASE
return caps
async def _get_pg_settings(
conn: asyncpg.Connection,
name: str,
) -> str:
return await conn.fetchval( # type: ignore
'SELECT setting FROM pg_settings WHERE name = $1', name)
async def _get_reserved_connections(conn: asyncpg.Connection, ) -> int:
rv = int(await _get_pg_settings(conn,
'superuser_reserved_connections'))
for name in [
'rds.rds_superuser_reserved_connections',
]:
value = await _get_pg_settings(conn, name)
if value:
rv += int(value)
return rv
conn = await rcluster.connect()
try:
user, dbname = await conn.fetchrow(
"SELECT current_user, current_database()")
cluster_type, superuser_name = await _get_cluster_type(conn)
max_connections = await _get_pg_settings(conn, 'max_connections')
capabilities = await _detect_capabilities(conn)
if t_id != buildmeta.get_default_tenant_id():
# GOTCHA: This tenant_id check cannot protect us from running
# multiple EdgeDB servers using the default tenant_id with
# different catalog versions on the same backend. However, that
# would fail during bootstrap in single-role/database mode.
if not capabilities & pgparams.BackendCapabilities.CREATE_ROLE:
raise ClusterError(
"The remote backend doesn't support CREATE ROLE; "
"multi-tenancy is disabled.")
if not capabilities & pgparams.BackendCapabilities.CREATE_DATABASE:
raise ClusterError(
"The remote backend doesn't support CREATE DATABASE; "
"multi-tenancy is disabled.")
parsed_ver = conn.get_server_version()
ver_string = conn.get_settings().server_version
instance_params = pgparams.BackendInstanceParams(
capabilities=capabilities,
version=pgparams.BackendVersion(
major=parsed_ver.major,
minor=parsed_ver.minor,
micro=parsed_ver.micro,
releaselevel=parsed_ver.releaselevel,
serial=parsed_ver.serial,
string=ver_string,
),
base_superuser=superuser_name,
max_connections=int(max_connections),
reserved_connections=await _get_reserved_connections(conn),
tenant_id=t_id,
)
finally:
await conn.close()
params.user = user
params.database = dbname
return cluster_type(
addrs[0],
params,
instance_params=instance_params,
ha_backend=ha_backend,
)
if args.instance_name:
logger.info(f'instance name: {args.instance_name!r}')
if devmode.is_in_dev_mode():
logger.info(f'development mode active')
logger.debug(
f"defaulting to the '{args.default_auth_method}' authentication method"
)
_init_parsers()
pg_cluster_init_by_us = False
pg_cluster_started_by_us = False
if args.tenant_id is None:
tenant_id = buildmeta.get_default_tenant_id()
else:
tenant_id = f'C{args.tenant_id}'
cluster: Union[pgcluster.Cluster, pgcluster.RemoteCluster]
default_runstate_dir: Optional[pathlib.Path]
if args.data_dir:
default_runstate_dir = args.data_dir
else:
default_runstate_dir = None
specified_runstate_dir: Optional[pathlib.Path]
if args.runstate_dir:
specified_runstate_dir = args.runstate_dir
elif args.bootstrap_only:
async def _get_cluster_mode(ctx: BootstrapContext) -> ClusterMode:
backend_params = ctx.cluster.get_runtime_params()
tenant_id = backend_params.tenant_id
# First, check the existence of EDGEDB_SUPERGROUP - the role which is
# usually created at the beginning of bootstrap.
is_default_tenant = tenant_id == buildmeta.get_default_tenant_id()
ignore_others = is_default_tenant and ctx.args.ignore_other_tenants
if is_default_tenant:
result = await ctx.conn.fetch('''
SELECT
r.rolname
FROM
pg_catalog.pg_roles AS r
WHERE
r.rolname LIKE ('%' || $1)
''', edbdef.EDGEDB_SUPERGROUP)
else:
result = await ctx.conn.fetch('''
SELECT
r.rolname
FROM
pg_catalog.pg_roles AS r
WHERE
r.rolname = $1
''', ctx.cluster.get_role_name(edbdef.EDGEDB_SUPERGROUP))
if result:
if not ignore_others:
# Either our tenant slot is occupied, or there is
# a default tenant present.
return ClusterMode.regular
# We were explicitly asked to ignore the other default tenant,
# so check specifically if our tenant slot is occupied and ignore
# the others.
# This mode is used for in-place upgrade.
for row in result:
rolname = row['rolname']
other_tenant_id = rolname[: -(len(edbdef.EDGEDB_SUPERGROUP) + 1)]
if other_tenant_id == tenant_id:
return ClusterMode.regular
# Then, check if the current database was bootstrapped in single-db mode.
has_instdata = await ctx.conn.fetch('''
SELECT
tablename
FROM
pg_catalog.pg_tables
WHERE
schemaname = 'edgedbinstdata'
AND tablename = 'instdata'
''')
if has_instdata:
return ClusterMode.single_database
# At last, check for single-role-bootstrapped instance by trying to find
# the EdgeDB System DB with the assumption that we are not running in
# single-db mode. If not found, this is a pristine backend cluster.
if is_default_tenant:
result = await ctx.conn.fetch(f'''
SELECT datname
FROM pg_database
WHERE datname LIKE '%' || $1
''', edbdef.EDGEDB_SYSTEM_DB)
else:
result = await ctx.conn.fetchval(f'''
SELECT datname
FROM pg_database
WHERE datname = $1
''', ctx.cluster.get_db_name(edbdef.EDGEDB_SYSTEM_DB))
if result:
if not ignore_others:
# Either our tenant slot is occupied, or there is
# a default tenant present.
return ClusterMode.single_role
# We were explicitly asked to ignore the other default tenant,
# so check specifically if our tenant slot is occupied and ignore
# the others.
# This mode is used for in-place upgrade.
for row in result:
dbname = row['datname']
other_tenant_id = dbname[: -(len(edbdef.EDGEDB_SYSTEM_DB) + 1)]
if other_tenant_id == tenant_id:
return ClusterMode.single_role
return ClusterMode.pristine
async def _check_catalog_compatibility(
ctx: BootstrapContext,
) -> asyncpg_con.Connection:
tenant_id = ctx.cluster.get_runtime_params().tenant_id
if ctx.mode == ClusterMode.single_database:
sys_db = await ctx.conn.fetchval(f'''
SELECT current_database()
FROM edgedbinstdata.instdata
WHERE key = '{edbdef.EDGEDB_TEMPLATE_DB}metadata'
AND json->>'tenant_id' = '{tenant_id}'
''')
else:
is_default_tenant = tenant_id == buildmeta.get_default_tenant_id()
if is_default_tenant:
sys_db = await ctx.conn.fetchval(f'''
SELECT datname
FROM pg_database
WHERE datname LIKE '%' || $1
ORDER BY
datname = $1,
datname DESC
LIMIT 1
''', edbdef.EDGEDB_SYSTEM_DB)
else:
sys_db = await ctx.conn.fetchval(f'''
SELECT datname
FROM pg_database
WHERE datname = $1
''', ctx.cluster.get_db_name(edbdef.EDGEDB_SYSTEM_DB))
if not sys_db:
raise errors.ConfigurationError(
'database instance is corrupt',
details=(
f'The database instance does not appear to have been fully '
f'initialized or has been corrupted.'
)
)
conn = await ctx.cluster.connect(database=sys_db)
try:
instancedata = await _get_instance_data(conn)
datadir_version = instancedata.get('version')
if datadir_version:
datadir_major = datadir_version.get('major')
expected_ver = buildmeta.get_version()
datadir_catver = instancedata.get('catver')
expected_catver = edbdef.EDGEDB_CATALOG_VERSION
status = dict(
data_catalog_version=datadir_catver,
expected_catalog_version=expected_catver,
)
if datadir_major != expected_ver.major:
for status_sink in ctx.args.status_sinks:
status_sink(f'INCOMPATIBLE={json.dumps(status)}')
raise errors.ConfigurationError(
'database instance incompatible with this version of EdgeDB',
details=(
f'The database instance was initialized with '
f'EdgeDB version {datadir_major}, '
f'which is incompatible with this version '
f'{expected_ver.major}'
),
hint=(
f'You need to recreate the instance and upgrade '
f'using dump/restore.'
)
)
if datadir_catver != expected_catver:
for status_sink in ctx.args.status_sinks:
status_sink(f'INCOMPATIBLE={json.dumps(status)}')
raise errors.ConfigurationError(
'database instance incompatible with this version of EdgeDB',
details=(
f'The database instance was initialized with '
f'EdgeDB format version {datadir_catver}, '
f'but this version of the server expects '
f'format version {expected_catver}'
),
hint=(
f'You need to recreate the instance and upgrade '
f'using dump/restore.'
)
)
except Exception:
await conn.close()
raise
return conn