def _delta_from_ddl(
ddl_stmt: qlast.DDLCommand,
*,
schema: s_schema.Schema,
modaliases: Mapping[Optional[str], str],
stdmode: bool=False,
internal_schema_mode: bool=False,
testmode: bool=False,
allow_dml_in_functions: bool=False,
schema_object_ids: Optional[
Mapping[Tuple[sn.Name, Optional[str]], uuid.UUID]
]=None,
compat_ver: Optional[verutils.Version] = None,
) -> Tuple[s_schema.Schema, sd.DeltaRoot]:
delta = sd.DeltaRoot()
context = sd.CommandContext(
modaliases=modaliases,
schema=schema,
stdmode=stdmode,
internal_schema_mode=internal_schema_mode,
testmode=testmode,
allow_dml_in_functions=allow_dml_in_functions,
schema_object_ids=schema_object_ids,
compat_ver=compat_ver,
)
with context(sd.DeltaRootContext(schema=schema, op=delta)):
cmd = cmd_from_ddl(
ddl_stmt,
schema=schema,
modaliases=modaliases,
context=context,
testmode=testmode,
)
schema = cmd.apply(schema, context)
if not stdmode:
if not isinstance(
cmd,
(sd.GlobalObjectCommand, sd.ExternalObjectCommand),
):
ver = schema.get_global(
s_ver.SchemaVersion, '__schema_version__')
ver_cmd = ver.init_delta_command(schema, sd.AlterObject)
ver_cmd.set_attribute_value('version', uuidgen.uuid1mc())
schema = ver_cmd.apply(schema, context)
delta.add(ver_cmd)
elif not isinstance(cmd, sd.ExternalObjectCommand):
gver = schema.get_global(
s_ver.GlobalSchemaVersion, '__global_schema_version__')
g_ver_cmd = gver.init_delta_command(schema, sd.AlterObject)
g_ver_cmd.set_attribute_value('version', uuidgen.uuid1mc())
schema = g_ver_cmd.apply(schema, context)
delta.add(g_ver_cmd)
delta.add(cmd)
delta.canonical = True
return schema, delta
def init_derived(self, schema, source, *qualifiers, as_copy,
merge_bases=None, replace_original=None,
mark_derived=False, attrs=None, dctx=None,
name=None, **kwargs):
if name is None:
derived_name = self.get_derived_name(
schema, source, *qualifiers, mark_derived=mark_derived)
else:
derived_name = name
if attrs is None:
attrs = {}
existing = schema.get(derived_name, None)
if existing is not None and replace_original:
derived = existing
schema = derived.update(schema, attrs)
else:
attrs['name'] = derived_name
if not attrs.get('id'):
attrs['id'] = uuidgen.uuid1mc()
fvals = dict(self.get_fields_values(schema))
fvals.update(attrs)
schema, derived = type(self).create_in_schema(schema, **fvals)
return schema, derived
async def _ensure_edgedb_database(
conn,
database,
owner,
*,
cluster,
builtin: bool = False,
objid: Optional[uuid.UUID] = None,
):
result = await _get_db_info(conn, database)
if not result:
logger.info(f'Creating database: ' f'{database}')
block = dbops.SQLBlock()
if objid is None:
objid = uuidgen.uuid1mc()
db = dbops.Database(
database,
owner=owner,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
dbops.CreateDatabase(db).generate(block)
await _execute_block(conn, block)
async def _create_edgedb_database(
ctx: BootstrapContext,
database: str,
owner: str,
*,
builtin: bool = False,
objid: Optional[uuid.UUID] = None,
) -> uuid.UUID:
logger.info(f'Creating database: {database}')
block = dbops.SQLBlock()
if objid is None:
objid = uuidgen.uuid1mc()
instance_params = ctx.cluster.get_runtime_params().instance_params
db = dbops.Database(
ctx.cluster.get_db_name(database),
owner=ctx.cluster.get_role_name(owner),
metadata=dict(
id=str(objid),
tenant_id=instance_params.tenant_id,
name=database,
builtin=builtin,
),
)
tpl_db = ctx.cluster.get_db_name(edbdef.EDGEDB_TEMPLATE_DB)
dbops.CreateDatabase(db, template=tpl_db).generate(block)
await _execute_block(ctx.conn, block)
return objid
async def _create_edgedb_template_database(
ctx: BootstrapContext,
) -> uuid.UUID:
backend_params = ctx.cluster.get_runtime_params()
have_c_utf8 = backend_params.has_c_utf8_locale
logger.info('Creating template database...')
block = dbops.SQLBlock()
dbid = uuidgen.uuid1mc()
db = dbops.Database(
ctx.cluster.get_db_name(edbdef.EDGEDB_TEMPLATE_DB),
owner=ctx.cluster.get_role_name(edbdef.EDGEDB_SUPERUSER),
is_template=True,
lc_collate='C',
lc_ctype='C.UTF-8' if have_c_utf8 else 'en_US.UTF-8',
encoding='UTF8',
metadata=dict(
id=str(dbid),
tenant_id=backend_params.tenant_id,
name=edbdef.EDGEDB_TEMPLATE_DB,
builtin=True,
),
)
dbops.CreateDatabase(db, template='template0').generate(block)
await _execute_block(ctx.conn, block)
return dbid
async def _create_edgedb_template_database(cluster, conn):
instance_params = cluster.get_runtime_params().instance_params
capabilities = instance_params.capabilities
have_c_utf8 = (capabilities & pgcluster.BackendCapabilities.C_UTF8_LOCALE)
logger.info('Creating template database...')
block = dbops.SQLBlock()
dbid = uuidgen.uuid1mc()
db = dbops.Database(
edbdef.EDGEDB_TEMPLATE_DB,
owner=edbdef.EDGEDB_SUPERUSER,
is_template=True,
template='template0',
lc_collate='C',
lc_ctype='C.UTF-8' if have_c_utf8 else 'en_US.UTF-8',
encoding='UTF8',
metadata=dict(
id=str(dbid),
builtin=True,
),
)
dbops.CreateDatabase(db).generate(block)
await _execute_block(conn, block)
return dbid
def make_schema_version(
schema: s_schema.Schema,
) -> Tuple[s_schema.Schema, s_ver.CreateSchemaVersion]:
sv = sn.UnqualName('__schema_version__')
schema_version = s_ver.CreateSchemaVersion(classname=sv)
schema_version.set_attribute_value('name', sv)
schema_version.set_attribute_value('version', uuidgen.uuid1mc())
schema_version.set_attribute_value('internal', True)
schema = sd.apply(schema_version, schema=schema)
return schema, schema_version
async def _ensure_edgedb_template_database(cluster, conn):
result = await _get_db_info(conn, edbdef.EDGEDB_TEMPLATE_DB)
if not result:
logger.info('Creating template database...')
block = dbops.SQLBlock()
dbid = uuidgen.uuid1mc()
db = dbops.Database(
edbdef.EDGEDB_TEMPLATE_DB,
owner=edbdef.EDGEDB_SUPERUSER,
is_template=True,
template='template0',
lc_collate='C',
lc_ctype=('C.UTF-8' if cluster.supports_c_utf8_locale()
else 'en_US.UTF-8'),
encoding='UTF8',
metadata=dict(
id=str(dbid),
builtin=True,
),
)
dbops.CreateDatabase(db).generate(block)
await _execute_block(conn, block)
return dbid
else:
alter = []
alter_owner = False
if not result['datistemplate']:
alter.append('IS_TEMPLATE = true')
if result['rolname'] != edbdef.EDGEDB_SUPERUSER:
alter_owner = True
if alter or alter_owner:
logger.info('Altering template database parameters...')
if alter:
await _execute(
conn,
'ALTER DATABASE {} WITH {}'.format(
edbdef.EDGEDB_TEMPLATE_DB,
' '.join(alter)))
if alter_owner:
await _execute(
conn,
'ALTER DATABASE {} OWNER TO {}'.format(
edbdef.EDGEDB_TEMPLATE_DB,
edbdef.EDGEDB_SUPERUSER))
return None
async def _ensure_edgedb_role(
ctx: BootstrapContext,
role_name: str,
*,
superuser: bool = False,
builtin: bool = False,
objid: Optional[uuid.UUID] = None,
) -> uuid.UUID:
member_of = set()
if superuser:
member_of.add(edbdef.EDGEDB_SUPERGROUP)
if objid is None:
objid = uuidgen.uuid1mc()
members = set()
login_role = ctx.cluster.get_connection_params().user
sup_role = ctx.cluster.get_role_name(edbdef.EDGEDB_SUPERUSER)
if login_role != sup_role:
members.add(login_role)
instance_params = ctx.cluster.get_runtime_params().instance_params
pg_role_name = ctx.cluster.get_role_name(role_name)
role = dbops.Role(
name=pg_role_name,
superuser=(superuser
and bool(instance_params.capabilities
& pgcluster.BackendCapabilities.SUPERUSER_ACCESS)),
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=[ctx.cluster.get_role_name(m) for m in member_of],
members=members,
metadata=dict(
id=str(objid),
name=role_name,
tenant_id=instance_params.tenant_id,
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(pg_role_name)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(ctx.conn, block)
return objid
async def _ensure_edgedb_database(conn, database, owner, *, cluster):
result = await _get_db_info(conn, database)
if not result:
logger.info(f'Creating database: ' f'{database}')
block = dbops.SQLBlock()
db = dbops.Database(
database,
owner=owner,
metadata=dict(id=str(uuidgen.uuid1mc()), ),
)
dbops.CreateDatabase(db).generate(block)
await _execute_block(conn, block)
async def _ensure_edgedb_role(
cluster,
conn,
username,
*,
membership=(),
is_superuser=False,
builtin=False,
objid=None,
) -> None:
membership = set(membership)
if is_superuser:
superuser_role = cluster.get_superuser_role()
if superuser_role:
# If the cluster is exposing an explicit superuser role,
# become a member of that instead of creating a superuser
# role directly.
membership.add(superuser_role)
superuser_flag = False
else:
superuser_flag = True
else:
superuser_flag = False
if objid is None:
objid = uuidgen.uuid1mc()
role = dbops.Role(
name=username,
is_superuser=superuser_flag,
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=membership,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(username)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
return objid
async def _ensure_edgedb_role(
cluster,
conn,
username,
*,
superuser=False,
builtin=False,
objid=None,
) -> None:
member_of = set()
if superuser:
member_of.add(edbdef.EDGEDB_SUPERGROUP)
if objid is None:
objid = uuidgen.uuid1mc()
members = set()
login_role = cluster.get_connection_params().user
if login_role != edbdef.EDGEDB_SUPERUSER:
members.add(login_role)
instance_params = cluster.get_runtime_params().instance_params
role = dbops.Role(
name=username,
superuser=(superuser
and bool(instance_params.capabilities
& pgcluster.BackendCapabilities.SUPERUSER_ACCESS)),
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=member_of,
members=members,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(username)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
return objid
async def _ensure_edgedb_user(conn, username, *, is_superuser=False):
role = dbops.Role(name=username,
is_superuser=is_superuser,
allow_login=True,
metadata=dict(
id=str(uuidgen.uuid1mc()),
__edgedb__='1',
))
create_role = dbops.CreateRole(role=role,
neg_conditions=[dbops.RoleExists(username)])
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
async def _set_edgedb_database_metadata(
ctx: BootstrapContext,
database: str,
*,
objid: Optional[uuid.UUID] = None,
) -> uuid.UUID:
logger.info(f'Configuring database: {database}')
block = dbops.SQLBlock()
if objid is None:
objid = uuidgen.uuid1mc()
instance_params = ctx.cluster.get_runtime_params().instance_params
db = dbops.Database(ctx.cluster.get_db_name(database))
metadata = dict(
id=str(objid),
tenant_id=instance_params.tenant_id,
name=database,
builtin=False,
)
dbops.SetMetadata(db, metadata).generate(block)
await _execute_block(ctx.conn, block)
return objid
async def _detect_capabilities(conn) -> BackendCapabilities:
caps = BackendCapabilities.NONE
try:
await conn.execute(f'ALTER SYSTEM SET foo = 10')
except asyncpg.InsufficientPrivilegeError:
configfile_access = False
except asyncpg.UndefinedObjectError:
configfile_access = True
else:
configfile_access = True
if configfile_access:
caps |= BackendCapabilities.CONFIGFILE_ACCESS
tx = conn.transaction()
await tx.start()
rname = str(uuidgen.uuid1mc())
try:
await conn.execute(f'CREATE ROLE "{rname}" WITH SUPERUSER')
except asyncpg.InsufficientPrivilegeError:
can_make_superusers = False
else:
can_make_superusers = True
finally:
await tx.rollback()
if can_make_superusers:
caps |= BackendCapabilities.SUPERUSER_ACCESS
coll = await conn.fetchval('''
SELECT collname FROM pg_collation
WHERE lower(replace(collname, '-', '')) = 'c.utf8' LIMIT 1;
''')
if coll is not None:
caps |= BackendCapabilities.C_UTF8_LOCALE
return caps
async def _create_edgedb_database(
conn,
database,
owner,
*,
builtin: bool = False,
objid: Optional[uuid.UUID] = None,
) -> uuid.UUID:
logger.info(f'Creating database: {database}')
block = dbops.SQLBlock()
if objid is None:
objid = uuidgen.uuid1mc()
db = dbops.Database(
database,
owner=owner,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
dbops.CreateDatabase(db).generate(block)
await _execute_block(conn, block)
return objid
def get_unique_random_name() -> str:
return base64.b64encode(uuidgen.uuid1mc().bytes).rstrip(b'=').decode()
async def _bootstrap(ctx: BootstrapContext) -> None:
args = ctx.args
cluster = ctx.cluster
backend_params = cluster.get_runtime_params()
if backend_params.instance_params.version < edbdef.MIN_POSTGRES_VERSION:
min_ver = '.'.join(str(v) for v in edbdef.MIN_POSTGRES_VERSION)
raise errors.ConfigurationError(
'unsupported backend',
details=(
f'EdgeDB requires PostgreSQL version {min_ver} or later, '
f'while the specified backend reports itself as '
f'{backend_params.instance_params.version.string}.'
)
)
if args.backend_capability_sets.must_be_absent:
caps = backend_params.instance_params.capabilities
disabled = []
for cap in args.backend_capability_sets.must_be_absent:
if caps & cap:
caps &= ~cap
disabled.append(cap)
if disabled:
logger.info(f"the following backend capabilities are disabled: "
f"{', '.join(str(cap.name) for cap in disabled)}")
cluster.overwrite_capabilities(caps)
_check_capabilities(ctx)
if backend_params.has_create_role:
superuser_uid = await _bootstrap_edgedb_super_roles(ctx)
else:
superuser_uid = uuidgen.uuid1mc()
in_dev_mode = devmode.is_in_dev_mode()
# Protect against multiple EdgeDB tenants from trying to bootstrap
# on the same cluster in devmode, as that is both a waste of resources
# and might result in broken stdlib cache.
if in_dev_mode:
bootstrap_lock = 0xEDB00001
await ctx.conn.execute('SELECT pg_advisory_lock($1)', bootstrap_lock)
if backend_params.has_create_database:
new_template_db_id = await _create_edgedb_template_database(ctx)
tpl_db = cluster.get_db_name(edbdef.EDGEDB_TEMPLATE_DB)
conn = await cluster.connect(database=tpl_db)
else:
new_template_db_id = uuidgen.uuid1mc()
try:
if backend_params.has_create_database:
tpl_ctx = ctx._replace(conn=conn)
conn.add_log_listener(_pg_log_listener)
else:
tpl_ctx = ctx
await _populate_misc_instance_data(tpl_ctx)
stdlib, config_spec, compiler = await _init_stdlib(
tpl_ctx,
testmode=args.testmode,
global_ids={
edbdef.EDGEDB_SUPERUSER: superuser_uid,
edbdef.EDGEDB_TEMPLATE_DB: new_template_db_id,
}
)
await _compile_sys_queries(
tpl_ctx,
stdlib.reflschema,
compiler,
config_spec,
)
schema = s_schema.FlatSchema()
schema = await _init_defaults(schema, compiler, tpl_ctx.conn)
finally:
if in_dev_mode:
await ctx.conn.execute(
'SELECT pg_advisory_unlock($1)',
bootstrap_lock,
)
if backend_params.has_create_database:
await conn.close()
if backend_params.has_create_database:
await _create_edgedb_database(
ctx,
edbdef.EDGEDB_SYSTEM_DB,
edbdef.EDGEDB_SUPERUSER,
builtin=True,
)
conn = await cluster.connect(
database=cluster.get_db_name(edbdef.EDGEDB_SYSTEM_DB))
try:
conn.add_log_listener(_pg_log_listener)
await _configure(
ctx._replace(conn=conn),
config_spec=config_spec,
schema=schema,
compiler=compiler,
)
finally:
await conn.close()
else:
await _configure(
ctx,
config_spec=config_spec,
schema=schema,
compiler=compiler,
)
if backend_params.has_create_database:
await _create_edgedb_database(
ctx,
edbdef.EDGEDB_SUPERUSER_DB,
edbdef.EDGEDB_SUPERUSER,
)
else:
await _set_edgedb_database_metadata(
ctx,
edbdef.EDGEDB_SUPERUSER_DB,
)
if (
backend_params.has_create_role
and args.default_database_user
and args.default_database_user != edbdef.EDGEDB_SUPERUSER
):
await _ensure_edgedb_role(
ctx,
args.default_database_user,
superuser=True,
)
def_role = ctx.cluster.get_role_name(args.default_database_user)
await _execute(ctx.conn, f"SET ROLE {qi(def_role)}")
if (
backend_params.has_create_database
and args.default_database
and args.default_database != edbdef.EDGEDB_SUPERUSER_DB
):
await _create_edgedb_database(
ctx,
args.default_database,
args.default_database_user or edbdef.EDGEDB_SUPERUSER,
)
async def _detect_capabilities(
conn: asyncpg.Connection, ) -> pgparams.BackendCapabilities:
caps = pgparams.BackendCapabilities.NONE
try:
cur_cluster_name = await conn.fetchval(f'''
SELECT
setting
FROM
pg_file_settings
WHERE
setting = 'cluster_name'
AND sourcefile = ((
SELECT setting
FROM pg_settings WHERE name = 'data_directory'
) || '/postgresql.auto.conf')
''')
except asyncpg.InsufficientPrivilegeError:
configfile_access = False
else:
try:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name = 'edgedb-test'
""")
except asyncpg.InsufficientPrivilegeError:
configfile_access = False
except asyncpg.InternalServerError as e:
# Stolon keeper symlinks postgresql.auto.conf to /dev/null
# making ALTER SYSTEM fail with InternalServerError,
# see https://github.com/sorintlab/stolon/pull/343
if 'could not fsync file "postgresql.auto.conf"' in e.args[0]:
configfile_access = False
else:
raise
else:
configfile_access = True
if cur_cluster_name:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name =
'{pgcommon.quote_literal(cur_cluster_name)}'
""")
else:
await conn.execute(f"""
ALTER SYSTEM SET cluster_name = DEFAULT
""")
if configfile_access:
caps |= pgparams.BackendCapabilities.CONFIGFILE_ACCESS
tx = conn.transaction()
await tx.start()
rname = str(uuidgen.uuid1mc())
try:
await conn.execute(f'CREATE ROLE "{rname}" WITH SUPERUSER')
except asyncpg.InsufficientPrivilegeError:
can_make_superusers = False
else:
can_make_superusers = True
finally:
await tx.rollback()
if can_make_superusers:
caps |= pgparams.BackendCapabilities.SUPERUSER_ACCESS
coll = await conn.fetchval('''
SELECT collname FROM pg_collation
WHERE lower(replace(collname, '-', '')) = 'c.utf8' LIMIT 1;
''')
if coll is not None:
caps |= pgparams.BackendCapabilities.C_UTF8_LOCALE
roles = await conn.fetchrow('''
SELECT rolcreaterole, rolcreatedb FROM pg_roles
WHERE rolname = (SELECT current_user);
''')
if roles['rolcreaterole']:
caps |= pgparams.BackendCapabilities.CREATE_ROLE
if roles['rolcreatedb']:
caps |= pgparams.BackendCapabilities.CREATE_DATABASE
return caps
async def _populate_misc_instance_data(
ctx: BootstrapContext,
) -> Dict[str, Any]:
commands = dbops.CommandGroup()
commands.add_commands([
dbops.CreateSchema(name='edgedbinstdata'),
dbops.CreateTable(dbops.Table(
name=('edgedbinstdata', 'instdata'),
columns=[
dbops.Column(
name='key',
type='text',
),
dbops.Column(
name='bin',
type='bytea',
),
dbops.Column(
name='text',
type='text',
),
dbops.Column(
name='json',
type='jsonb',
),
],
constraints=[
dbops.PrimaryKey(
table_name=('edgedbinstdata', 'instdata'),
columns=['key'],
),
],
))
])
block = dbops.PLTopBlock()
commands.generate(block)
await _execute_block(ctx.conn, block)
mock_auth_nonce = scram.generate_nonce()
json_instance_data = {
'version': dict(buildmeta.get_version_dict()),
'catver': edbdef.EDGEDB_CATALOG_VERSION,
'mock_auth_nonce': mock_auth_nonce,
}
await _store_static_json_cache(
ctx,
'instancedata',
json.dumps(json_instance_data),
)
backend_params = ctx.cluster.get_runtime_params()
instance_params = backend_params.instance_params
await _store_static_json_cache(
ctx,
'backend_instance_params',
json.dumps(instance_params._asdict()),
)
if not backend_params.has_create_role:
json_single_role_metadata = {
'id': str(uuidgen.uuid1mc()),
'name': edbdef.EDGEDB_SUPERUSER,
'tenant_id': backend_params.tenant_id,
'builtin': False,
}
await _store_static_json_cache(
ctx,
'single_role_metadata',
json.dumps(json_single_role_metadata),
)
if not backend_params.has_create_database:
await _store_static_json_cache(
ctx,
f'{edbdef.EDGEDB_TEMPLATE_DB}metadata',
json.dumps({}),
)
await _store_static_json_cache(
ctx,
f'{edbdef.EDGEDB_SYSTEM_DB}metadata',
json.dumps({}),
)
return json_instance_data