def _compile_and_apply_ddl_command(self, ctx: CompileContext, cmd):
current_tx = ctx.state.current_tx()
schema = current_tx.get_schema()
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(cmd)
# Do a dry-run on test_schema to canonicalize
# the schema delta-commands.
test_schema = schema
context = self._new_delta_context(ctx)
cmd.apply(test_schema, context=context)
cmd.canonical = True
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = self._process_delta(ctx, cmd, schema)
if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)):
block = pg_dbops.SQLBlock()
else:
block = pg_dbops.PLTopBlock()
plan.generate(block)
sql = block.to_string().encode('utf-8')
current_tx.update_schema(schema)
if debug.flags.delta_execute:
debug.header('Delta Script')
debug.dump_code(sql, lexer='sql')
return dbstate.DDLQuery(sql=(sql, ))
async def _populate_misc_instance_data(cluster, conn):
commands = dbops.CommandGroup()
commands.add_commands([
dbops.CreateSchema(name='edgedbinstdata'),
])
block = dbops.PLTopBlock()
commands.generate(block)
await _execute_block(conn, block)
mock_auth_nonce = scram.generate_nonce()
json_instance_data = {
'version': dict(buildmeta.get_version_dict()),
'catver': edbdef.EDGEDB_CATALOG_VERSION,
'mock_auth_nonce': mock_auth_nonce,
}
await _store_static_json_cache(
cluster,
'instancedata',
json.dumps(json_instance_data),
)
return json_instance_data
async def _make_stdlib(testmode: bool):
schema = s_schema.Schema()
current_block = None
std_texts = []
for modname in s_schema.STD_LIB + ('stdgraphql', ):
std_texts.append(s_std.get_std_module_text(modname))
if testmode:
std_texts.append(s_std.get_std_module_text('_testmode'))
ddl_text = '\n'.join(std_texts)
for ddl_cmd in edgeql.parse_block(ddl_text):
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
schema=schema,
modaliases={None: 'std'},
stdmode=True)
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(delta_command)
# Do a dry-run on test_schema to canonicalize
# the schema delta-commands.
test_schema = schema
context = sd.CommandContext()
context.stdmode = True
delta_command.apply(test_schema, context=context)
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = _process_delta(delta_command, schema)
if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)):
if (current_block is not None
and not isinstance(current_block, dbops.SQLBlock)):
raise errors.QueryError(
'cannot mix DATABASE commands with regular DDL '
'commands in a single block')
if current_block is None:
current_block = dbops.SQLBlock()
else:
if (current_block is not None
and not isinstance(current_block, dbops.PLTopBlock)):
raise errors.QueryError(
'cannot mix DATABASE commands with regular DDL '
'commands in a single block')
if current_block is None:
current_block = dbops.PLTopBlock()
plan.generate(current_block)
sql_text = current_block.to_string()
return schema, sql_text
async def _populate_misc_instance_data(cluster, conn):
commands = dbops.CommandGroup()
commands.add_commands([
dbops.CreateSchema(name='edgedbinstdata'),
dbops.CreateTable(
dbops.Table(
name=('edgedbinstdata', 'instdata'),
columns=[
dbops.Column(
name='key',
type='text',
),
dbops.Column(
name='bin',
type='bytea',
),
dbops.Column(
name='text',
type='text',
),
dbops.Column(
name='json',
type='jsonb',
),
],
constraints=[
dbops.PrimaryKey(
table_name=('edgedbinstdata', 'instdata'),
columns=['key'],
),
],
))
])
block = dbops.PLTopBlock()
commands.generate(block)
await _execute_block(conn, block)
mock_auth_nonce = scram.generate_nonce()
json_instance_data = {
'version': dict(buildmeta.get_version_dict()),
'catver': edbdef.EDGEDB_CATALOG_VERSION,
'mock_auth_nonce': mock_auth_nonce,
}
await _store_static_json_cache(
cluster,
'instancedata',
json.dumps(json_instance_data),
)
instance_params = cluster.get_runtime_params().instance_params
await _store_static_json_cache(
cluster,
'backend_instance_params',
json.dumps(instance_params._asdict()),
)
return json_instance_data
async def _amend_stdlib(
ddl_text: str,
stdlib: StdlibBits,
) -> Tuple[StdlibBits, str]:
schema = stdlib.stdschema
reflschema = stdlib.reflschema
topblock = dbops.PLTopBlock()
plans = []
context = sd.CommandContext()
context.stdmode = True
for ddl_cmd in edgeql.parse_block(ddl_text):
assert isinstance(ddl_cmd, qlast.DDLCommand)
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
modaliases={},
schema=schema,
stdmode=True)
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(delta_command)
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = _process_delta(delta_command, schema)
reflschema = delta_command.apply(reflschema, context)
plan.generate(topblock)
plans.append(plan)
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=reflschema,
schema_class_layout=stdlib.classlayout,
)
compilerctx = edbcompiler.new_compiler_context(
schema,
bootstrap_mode=True,
)
compilerctx = edbcompiler.new_compiler_context(schema)
for plan in plans:
compiler._compile_schema_storage_in_delta(
ctx=compilerctx,
delta=plan,
block=topblock,
)
sqltext = topblock.to_string()
return stdlib._replace(stdschema=schema, reflschema=reflschema), sqltext
async def _ensure_edgedb_role(
ctx: BootstrapContext,
role_name: str,
*,
superuser: bool = False,
builtin: bool = False,
objid: Optional[uuid.UUID] = None,
) -> uuid.UUID:
member_of = set()
if superuser:
member_of.add(edbdef.EDGEDB_SUPERGROUP)
if objid is None:
objid = uuidgen.uuid1mc()
members = set()
login_role = ctx.cluster.get_connection_params().user
sup_role = ctx.cluster.get_role_name(edbdef.EDGEDB_SUPERUSER)
if login_role != sup_role:
members.add(login_role)
instance_params = ctx.cluster.get_runtime_params().instance_params
pg_role_name = ctx.cluster.get_role_name(role_name)
role = dbops.Role(
name=pg_role_name,
superuser=(superuser
and bool(instance_params.capabilities
& pgcluster.BackendCapabilities.SUPERUSER_ACCESS)),
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=[ctx.cluster.get_role_name(m) for m in member_of],
members=members,
metadata=dict(
id=str(objid),
name=role_name,
tenant_id=instance_params.tenant_id,
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(pg_role_name)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(ctx.conn, block)
return objid
async def _ensure_edgedb_role(
cluster,
conn,
username,
*,
membership=(),
is_superuser=False,
builtin=False,
objid=None,
) -> None:
membership = set(membership)
if is_superuser:
superuser_role = cluster.get_superuser_role()
if superuser_role:
# If the cluster is exposing an explicit superuser role,
# become a member of that instead of creating a superuser
# role directly.
membership.add(superuser_role)
superuser_flag = False
else:
superuser_flag = True
else:
superuser_flag = False
if objid is None:
objid = uuidgen.uuid1mc()
role = dbops.Role(
name=username,
is_superuser=superuser_flag,
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=membership,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(username)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
return objid
async def _ensure_edgedb_role(
cluster,
conn,
username,
*,
superuser=False,
builtin=False,
objid=None,
) -> None:
member_of = set()
if superuser:
member_of.add(edbdef.EDGEDB_SUPERGROUP)
if objid is None:
objid = uuidgen.uuid1mc()
members = set()
login_role = cluster.get_connection_params().user
if login_role != edbdef.EDGEDB_SUPERUSER:
members.add(login_role)
instance_params = cluster.get_runtime_params().instance_params
role = dbops.Role(
name=username,
superuser=(superuser
and bool(instance_params.capabilities
& pgcluster.BackendCapabilities.SUPERUSER_ACCESS)),
allow_login=True,
allow_createdb=True,
allow_createrole=True,
membership=member_of,
members=members,
metadata=dict(
id=str(objid),
builtin=builtin,
),
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(username)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
return objid
async def _configure(schema, conn, cluster, *, insecure=False, testmode=False):
scripts = []
if not testmode:
memory_kb = psutil.virtual_memory().total // 1024
settings = {
'shared_buffers': f'"{int(memory_kb * 0.2)}kB"',
'effective_cache_size': f'"{int(memory_kb * 0.5)}kB"',
'query_work_mem': f'"{6 * (2 ** 10)}kB"',
}
for setting, value in settings.items():
scripts.append(f'''
CONFIGURE SYSTEM SET {setting} := {value};
''')
else:
settings = {}
if insecure:
scripts.append('''
CONFIGURE SYSTEM INSERT Auth {
priority := 0,
method := (INSERT Trust),
};
''')
config_spec = config.get_settings()
for script in scripts:
_, sql = compiler.compile_bootstrap_script(
schema, schema, script, single_statement=True)
if debug.flags.bootstrap:
debug.header('Bootstrap')
debug.dump_code(sql, lexer='sql')
config_op_data = await conn.fetchval(sql)
if config_op_data is not None and isinstance(config_op_data, str):
config_op = config.Operation.from_json(config_op_data)
settings = config_op.apply(config_spec, immutables.Map())
config_json = config.to_json(config_spec, settings)
block = dbops.PLTopBlock()
dbops.UpdateMetadata(
dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB),
{'sysconfig': json.loads(config_json)},
).generate(block)
await _execute_block(conn, block)
async def _configure(
ctx: BootstrapContext,
config_spec: config.Spec,
schema: s_schema.Schema,
compiler: edbcompiler.Compiler,
) -> None:
settings: Mapping[str, config.SettingValue] = {}
config_json = config.to_json(config_spec, settings, include_source=False)
block = dbops.PLTopBlock()
metadata = {'sysconfig': json.loads(config_json)}
if ctx.cluster.get_runtime_params().has_create_database:
dbops.UpdateMetadata(
dbops.Database(
name=ctx.cluster.get_db_name(edbdef.EDGEDB_SYSTEM_DB)
),
metadata,
).generate(block)
else:
dbops.UpdateSingleDBMetadata(
edbdef.EDGEDB_SYSTEM_DB, metadata,
).generate(block)
await _execute_block(ctx.conn, block)
backend_params = ctx.cluster.get_runtime_params()
for setname in config_spec:
setting = config_spec[setname]
if (
setting.backend_setting
and setting.default is not None
and (
# Do not attempt to run CONFIGURE INSTANCE on
# backends that don't support it.
# TODO: this should be replaced by instance-wide
# emulation at backend connection time.
backend_params.has_configfile_access
)
):
if isinstance(setting.default, statypes.Duration):
val = f'<std::duration>"{setting.default.to_iso8601()}"'
else:
val = repr(setting.default)
script = f'''
CONFIGURE INSTANCE SET {setting.name} := {val};
'''
schema, sql = compile_bootstrap_script(compiler, schema, script)
await _execute_ddl(ctx.conn, sql)
async def _configure(
schema: s_schema.Schema,
compiler: edbcompiler.Compiler,
conn: asyncpg_con.Connection,
cluster: pgcluster.BaseCluster,
*,
insecure: bool = False,
) -> None:
config_spec = config.get_settings()
scripts = []
settings: Mapping[str, config.SettingValue] = {}
if insecure:
scripts.append('''
CONFIGURE SYSTEM INSERT Auth {
priority := 0,
method := (INSERT Trust),
};
''')
for script in scripts:
_, sql = compile_bootstrap_script(
compiler,
schema,
script,
single_statement=True,
)
if debug.flags.bootstrap:
debug.header('Bootstrap')
debug.dump_code(sql, lexer='sql')
config_op_data = await conn.fetchval(sql)
if config_op_data is not None and isinstance(config_op_data, str):
config_op = config.Operation.from_json(config_op_data)
settings = config_op.apply(config_spec, immutables.Map())
config_json = config.to_json(config_spec, settings, include_source=False)
block = dbops.PLTopBlock()
dbops.UpdateMetadata(
dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB),
{
'sysconfig': json.loads(config_json)
},
).generate(block)
await _execute_block(conn, block)
async def _ensure_edgedb_user(conn, username, *, is_superuser=False):
role = dbops.Role(name=username,
is_superuser=is_superuser,
allow_login=True,
metadata=dict(
id=str(uuidgen.uuid1mc()),
__edgedb__='1',
))
create_role = dbops.CreateRole(role=role,
neg_conditions=[dbops.RoleExists(username)])
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(conn, block)
async def _ensure_edgedb_supergroup(
ctx: BootstrapContext,
role_name: str,
*,
member_of: Iterable[str] = (),
members: Iterable[str] = (),
) -> None:
member_of = set(member_of)
instance_params = ctx.cluster.get_runtime_params().instance_params
superuser_role = instance_params.base_superuser
if superuser_role:
# If the cluster is exposing an explicit superuser role,
# become a member of that instead of creating a superuser
# role directly.
member_of.add(superuser_role)
pg_role_name = ctx.cluster.get_role_name(role_name)
role = dbops.Role(
name=pg_role_name,
superuser=bool(instance_params.capabilities
& pgcluster.BackendCapabilities.SUPERUSER_ACCESS),
allow_login=False,
allow_createdb=True,
allow_createrole=True,
membership=member_of,
members=members,
)
create_role = dbops.CreateRole(
role,
neg_conditions=[dbops.RoleExists(pg_role_name)],
)
block = dbops.PLTopBlock()
create_role.generate(block)
await _execute_block(ctx.conn, block)
async def _init_stdlib(cluster, conn, testmode, global_ids):
in_dev_mode = devmode.is_in_dev_mode()
specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO')
if specified_cache_dir:
cache_dir = pathlib.Path(specified_cache_dir)
else:
cache_dir = None
stdlib_cache = 'backend-stdlib.pickle'
tpldbdump_cache = 'backend-tpldbdump.sql'
src_hash = buildmeta.hash_dirs(
buildmeta.get_cache_src_dirs(),
extra_files=[__file__],
)
stdlib = buildmeta.read_data_cache(src_hash,
stdlib_cache,
source_dir=cache_dir)
tpldbdump = buildmeta.read_data_cache(src_hash,
tpldbdump_cache,
source_dir=cache_dir,
pickled=False)
if stdlib is None:
logger.info('Compiling the standard library...')
stdlib = await _make_stdlib(in_dev_mode or testmode, global_ids)
logger.info('Creating the necessary PostgreSQL extensions...')
await metaschema.create_pg_extensions(conn)
if tpldbdump is None:
logger.info('Populating internal SQL structures...')
await metaschema.bootstrap(conn)
logger.info('Executing the standard library...')
await _execute_ddl(conn, stdlib.sqltext)
if in_dev_mode or specified_cache_dir:
tpldbdump = cluster.dump_database(
edbdef.EDGEDB_TEMPLATE_DB,
exclude_schemas=['edgedbinstdata', 'edgedbext'],
)
# Excluding the "edgedbext" schema above apparently
# doesn't apply to extensions created in that schema,
# so we have to resort to commenting out extension
# statements in the dump.
tpldbdump = re.sub(
rb'^(CREATE|COMMENT ON) EXTENSION.*$',
rb'-- \g<0>',
tpldbdump,
flags=re.MULTILINE,
)
global_metadata = await conn.fetchval(f'''\
SELECT edgedb.shobj_metadata(
(SELECT oid FROM pg_database
WHERE datname = {ql(edbdef.EDGEDB_TEMPLATE_DB)}),
'pg_database'
)''')
pl_block = dbops.PLTopBlock()
dbops.SetMetadata(
dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB),
json.loads(global_metadata),
).generate(pl_block)
tpldbdump += b'\n' + pl_block.to_string().encode('utf-8')
buildmeta.write_data_cache(
tpldbdump,
src_hash,
tpldbdump_cache,
pickled=False,
target_dir=cache_dir,
)
buildmeta.write_data_cache(
stdlib,
src_hash,
stdlib_cache,
target_dir=cache_dir,
)
else:
logger.info('Initializing the standard library...')
await metaschema._execute_sql_script(conn, tpldbdump.decode('utf-8'))
# When we restore a database from a dump, OIDs for non-system
# Postgres types might get skewed as they are not part of the dump.
# A good example of that is `std::bigint` which is implemented as
# a custom domain type. The OIDs are stored under
# `schema::Object.backend_id` property and are injected into
# array query arguments.
#
# The code below re-syncs backend_id properties of EdgeDB builtin
# types with the actual OIDs in the DB.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
UPDATE schema::ScalarType
FILTER .builtin AND NOT (.abstract ?? False)
SET {
backend_id := sys::_get_pg_type_for_scalar_type(.id)
}
''',
expected_cardinality_one=False,
single_statement=True,
)
await conn.execute(sql)
if not in_dev_mode and testmode:
# Running tests on a production build.
stdlib, testmode_sql = await _amend_stdlib(
s_std.get_std_module_text('_testmode'),
stdlib,
)
await conn.execute(testmode_sql)
await metaschema.generate_support_views(
conn,
stdlib.reflschema,
)
# Make sure that schema backend_id properties are in sync with
# the database.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
SELECT schema::ScalarType {
id,
backend_id,
} FILTER .builtin AND NOT (.abstract ?? False);
''',
expected_cardinality_one=False,
single_statement=True,
)
schema = stdlib.stdschema
typemap = await conn.fetchval(sql)
for entry in json.loads(typemap):
t = schema.get_by_id(uuidgen.UUID(entry['id']))
schema = t.set_field_value(schema, 'backend_id', entry['backend_id'])
stdlib = stdlib._replace(stdschema=schema)
await _store_static_bin_cache(
cluster,
'stdschema',
pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'reflschema',
pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'global_schema',
pickle.dumps(stdlib.global_schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'classlayout',
pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_text_cache(
cluster,
'local_intro_query',
stdlib.local_intro_query,
)
await _store_static_text_cache(
cluster,
'global_intro_query',
stdlib.global_intro_query,
)
await metaschema.generate_support_views(conn, stdlib.reflschema)
await metaschema.generate_support_functions(conn, stdlib.reflschema)
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
await metaschema.generate_more_support_functions(conn, compiler,
stdlib.reflschema,
testmode)
return stdlib, compiler
async def _make_stdlib(testmode: bool, global_ids) -> StdlibBits:
schema = s_schema.ChainedSchema(
s_schema.FlatSchema(),
s_schema.FlatSchema(),
s_schema.FlatSchema(),
)
schema, _ = s_mod.Module.create_in_schema(
schema,
name=sn.UnqualName('__derived__'),
)
current_block = dbops.PLTopBlock()
std_texts = []
for modname in s_schema.STD_SOURCES:
std_texts.append(s_std.get_std_module_text(modname))
if testmode:
std_texts.append(s_std.get_std_module_text(sn.UnqualName('_testmode')))
ddl_text = '\n'.join(std_texts)
types: Set[uuid.UUID] = set()
std_plans: List[sd.Command] = []
for ddl_cmd in edgeql.parse_block(ddl_text):
assert isinstance(ddl_cmd, qlast.DDLCommand)
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
modaliases={},
schema=schema,
stdmode=True)
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(delta_command)
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = _process_delta(delta_command, schema)
std_plans.append(delta_command)
types.update(plan.new_types)
plan.generate(current_block)
_, schema_version = s_std.make_schema_version(schema)
schema, plan = _process_delta(schema_version, schema)
std_plans.append(schema_version)
plan.generate(current_block)
stdglobals = '\n'.join([
f'''CREATE SUPERUSER ROLE {edbdef.EDGEDB_SUPERUSER} {{
SET id := <uuid>'{global_ids[edbdef.EDGEDB_SUPERUSER]}'
}};''',
])
schema = await _execute_edgeql_ddl(schema, stdglobals)
_, global_schema_version = s_std.make_global_schema_version(schema)
schema, plan = _process_delta(global_schema_version, schema)
std_plans.append(global_schema_version)
plan.generate(current_block)
reflection = s_refl.generate_structure(schema)
reflschema, reflplan = _process_delta(reflection.intro_schema_delta,
schema)
assert current_block is not None
reflplan.generate(current_block)
subblock = current_block.add_block()
compiler = edbcompiler.new_compiler(
std_schema=schema.get_top_schema(),
reflection_schema=reflschema.get_top_schema(),
schema_class_layout=reflection.class_layout, # type: ignore
)
compilerctx = edbcompiler.new_compiler_context(
user_schema=reflschema.get_top_schema(),
global_schema=schema.get_global_schema(),
bootstrap_mode=True,
)
for std_plan in std_plans:
compiler._compile_schema_storage_in_delta(
ctx=compilerctx,
delta=std_plan,
block=subblock,
)
compilerctx = edbcompiler.new_compiler_context(
user_schema=reflschema.get_top_schema(),
global_schema=schema.get_global_schema(),
bootstrap_mode=True,
internal_schema_mode=True,
)
compiler._compile_schema_storage_in_delta(
ctx=compilerctx,
delta=reflection.intro_schema_delta,
block=subblock,
)
sqltext = current_block.to_string()
compilerctx = edbcompiler.new_compiler_context(
user_schema=reflschema.get_top_schema(),
global_schema=schema.get_global_schema(),
schema_reflection_mode=True,
output_format=edbcompiler.IoFormat.JSON_ELEMENTS,
)
# The introspection query bits are returned in chunks
# because it's a large UNION and we currently generate SQL
# that is much harder for Posgres to plan as opposed to a
# straight flat UNION.
sql_intro_local_parts = []
sql_intro_global_parts = []
for intropart in reflection.local_intro_parts:
sql_intro_local_parts.append(
compile_single_query(
intropart,
compiler=compiler,
compilerctx=compilerctx,
), )
for intropart in reflection.global_intro_parts:
sql_intro_global_parts.append(
compile_single_query(
intropart,
compiler=compiler,
compilerctx=compilerctx,
), )
local_intro_sql = ' UNION ALL '.join(sql_intro_local_parts)
local_intro_sql = f'''
WITH intro(c) AS ({local_intro_sql})
SELECT json_agg(intro.c) FROM intro
'''
global_intro_sql = ' UNION ALL '.join(sql_intro_global_parts)
global_intro_sql = f'''
WITH intro(c) AS ({global_intro_sql})
SELECT json_agg(intro.c) FROM intro
'''
return StdlibBits(
stdschema=schema.get_top_schema(),
reflschema=reflschema.get_top_schema(),
global_schema=schema.get_global_schema(),
sqltext=sqltext,
types=types,
classlayout=reflection.class_layout,
local_intro_query=local_intro_sql,
global_intro_query=global_intro_sql,
)
async def _init_stdlib(
ctx: BootstrapContext,
testmode: bool,
global_ids: Mapping[str, uuid.UUID],
) -> Tuple[StdlibBits, config.Spec, edbcompiler.Compiler]:
in_dev_mode = devmode.is_in_dev_mode()
conn = ctx.conn
cluster = ctx.cluster
specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO')
if not specified_cache_dir:
cache_dir = None
else:
cache_dir = pathlib.Path(specified_cache_dir)
stdlib_cache = f'backend-stdlib.pickle'
tpldbdump_cache = f'backend-tpldbdump.sql'
src_hash = buildmeta.hash_dirs(
buildmeta.get_cache_src_dirs(), extra_files=[__file__],
)
stdlib = buildmeta.read_data_cache(
src_hash, stdlib_cache, source_dir=cache_dir)
tpldbdump = buildmeta.read_data_cache(
src_hash, tpldbdump_cache, source_dir=cache_dir, pickled=False)
if stdlib is None:
logger.info('Compiling the standard library...')
stdlib = await _make_stdlib(ctx, in_dev_mode or testmode, global_ids)
logger.info('Creating the necessary PostgreSQL extensions...')
await metaschema.create_pg_extensions(conn)
config_spec = config.load_spec_from_schema(stdlib.stdschema)
config.set_settings(config_spec)
if tpldbdump is None:
logger.info('Populating internal SQL structures...')
await metaschema.bootstrap(conn, config_spec)
logger.info('Executing the standard library...')
await _execute_ddl(conn, stdlib.sqltext)
if in_dev_mode or specified_cache_dir:
tpl_db_name = edbdef.EDGEDB_TEMPLATE_DB
tpl_pg_db_name = cluster.get_db_name(tpl_db_name)
tpl_pg_db_name_dyn = (
f"edgedb.get_database_backend_name({ql(tpl_db_name)})")
tpldbdump = await cluster.dump_database(
tpl_pg_db_name,
exclude_schemas=['edgedbinstdata', 'edgedbext'],
dump_object_owners=False,
)
# Excluding the "edgedbext" schema above apparently
# doesn't apply to extensions created in that schema,
# so we have to resort to commenting out extension
# statements in the dump.
tpldbdump = re.sub(
rb'^(CREATE|COMMENT ON) EXTENSION.*$',
rb'-- \g<0>',
tpldbdump,
flags=re.MULTILINE,
)
global_metadata = await conn.fetchval(
f'SELECT edgedb.get_database_metadata({ql(tpl_db_name)})',
)
global_metadata = json.loads(global_metadata)
pl_block = dbops.PLTopBlock()
set_metadata_text = dbops.SetMetadata(
dbops.Database(name='__dummy_placeholder_database__'),
global_metadata,
).code(pl_block)
set_metadata_text = set_metadata_text.replace(
'__dummy_placeholder_database__',
f"' || quote_ident({tpl_pg_db_name_dyn}) || '",
)
set_single_db_metadata_text = dbops.SetSingleDBMetadata(
edbdef.EDGEDB_TEMPLATE_DB, global_metadata
).code(pl_block)
pl_block.add_command(textwrap.dedent(f"""\
IF (edgedb.get_backend_capabilities()
& {int(params.BackendCapabilities.CREATE_DATABASE)}) != 0
THEN
{textwrap.indent(set_metadata_text, ' ')}
ELSE
{textwrap.indent(set_single_db_metadata_text, ' ')}
END IF
"""))
text = pl_block.to_string()
tpldbdump += b'\n' + text.encode('utf-8')
buildmeta.write_data_cache(
tpldbdump,
src_hash,
tpldbdump_cache,
pickled=False,
target_dir=cache_dir,
)
buildmeta.write_data_cache(
stdlib,
src_hash,
stdlib_cache,
target_dir=cache_dir,
)
else:
logger.info('Initializing the standard library...')
await metaschema._execute_sql_script(conn, tpldbdump.decode('utf-8'))
# Restore the search_path as the dump might have altered it.
await conn.execute(
"SELECT pg_catalog.set_config('search_path', 'edgedb', false)")
if not in_dev_mode and testmode:
# Running tests on a production build.
stdlib, testmode_sql = await _amend_stdlib(
ctx,
s_std.get_std_module_text(sn.UnqualName('_testmode')),
stdlib,
)
await conn.execute(testmode_sql)
# _testmode includes extra config settings, so make sure
# those are picked up.
config_spec = config.load_spec_from_schema(stdlib.stdschema)
config.set_settings(config_spec)
# Make sure that schema backend_id properties are in sync with
# the database.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
SELECT schema::ScalarType {
id,
backend_id,
} FILTER .builtin AND NOT (.abstract ?? False);
''',
expected_cardinality_one=False,
single_statement=True,
)
schema = stdlib.stdschema
typemap = await conn.fetchval(sql)
for entry in json.loads(typemap):
t = schema.get_by_id(uuidgen.UUID(entry['id']))
schema = t.set_field_value(
schema, 'backend_id', entry['backend_id'])
stdlib = stdlib._replace(stdschema=schema)
await _store_static_bin_cache(
ctx,
'stdschema',
pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
ctx,
'reflschema',
pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
ctx,
'global_schema',
pickle.dumps(stdlib.global_schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
ctx,
'classlayout',
pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_text_cache(
ctx,
'local_intro_query',
stdlib.local_intro_query,
)
await _store_static_text_cache(
ctx,
'global_intro_query',
stdlib.global_intro_query,
)
await metaschema.generate_support_views(
conn, stdlib.reflschema, cluster.get_runtime_params()
)
await metaschema.generate_support_functions(conn, stdlib.reflschema)
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
await metaschema.generate_more_support_functions(
conn, compiler, stdlib.reflschema, testmode)
if tpldbdump is not None:
# When we restore a database from a dump, OIDs for non-system
# Postgres types might get skewed as they are not part of the dump.
# A good example of that is `std::bigint` which is implemented as
# a custom domain type. The OIDs are stored under
# `schema::Object.backend_id` property and are injected into
# array query arguments.
#
# The code below re-syncs backend_id properties of EdgeDB builtin
# types with the actual OIDs in the DB.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
UPDATE schema::Type
FILTER
.builtin
AND NOT (.abstract ?? False)
AND schema::Type IS schema::ScalarType | schema::Tuple
SET {
backend_id := sys::_get_pg_type_for_edgedb_type(
.id,
<uuid>{}
)
}
''',
expected_cardinality_one=False,
single_statement=True,
)
await conn.execute(sql)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
UPDATE schema::Array
FILTER
.builtin
AND NOT (.abstract ?? False)
SET {
backend_id := sys::_get_pg_type_for_edgedb_type(
.id,
.element_type.id,
)
}
''',
expected_cardinality_one=False,
single_statement=True,
)
await conn.execute(sql)
await _store_static_json_cache(
ctx,
'configspec',
config.spec_to_json(config_spec),
)
return stdlib, config_spec, compiler
async def _make_stdlib(
testmode: bool) -> Tuple[s_schema.Schema, str, Set[uuid.UUID]]:
schema = s_schema.Schema()
schema, _ = s_mod.Module.create_in_schema(schema, name='__derived__')
schema = s_pseudo.populate_types(schema)
current_block = None
std_texts = []
for modname in s_schema.STD_LIB + ('stdgraphql', ):
std_texts.append(s_std.get_std_module_text(modname))
if testmode:
std_texts.append(s_std.get_std_module_text('_testmode'))
ddl_text = '\n'.join(std_texts)
new_types: Set[uuid.UUID] = set()
for ddl_cmd in edgeql.parse_block(ddl_text):
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
modaliases={},
schema=schema,
stdmode=True)
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(delta_command)
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = _process_delta(delta_command, schema)
if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)):
if (current_block is not None
and not isinstance(current_block, dbops.SQLBlock)):
raise errors.QueryError(
'cannot mix DATABASE commands with regular DDL '
'commands in a single block')
if current_block is None:
current_block = dbops.SQLBlock()
else:
new_types.update(plan.new_types)
if (current_block is not None
and not isinstance(current_block, dbops.PLTopBlock)):
raise errors.QueryError(
'cannot mix DATABASE commands with regular DDL '
'commands in a single block')
if current_block is None:
current_block = dbops.PLTopBlock()
plan.generate(current_block)
assert current_block is not None
sql_text = current_block.to_string()
mods = {
mod.get_name(schema)
for mod in schema.get_modules() if mod.get_builtin(schema)
}
if mods != s_schema.STD_MODULES:
raise errors.SchemaError(
f'modules {s_schema.STD_MODULES - mods} are not marked as builtin')
return schema, sql_text, new_types
async def _populate_misc_instance_data(
ctx: BootstrapContext,
) -> Dict[str, Any]:
commands = dbops.CommandGroup()
commands.add_commands([
dbops.CreateSchema(name='edgedbinstdata'),
dbops.CreateTable(dbops.Table(
name=('edgedbinstdata', 'instdata'),
columns=[
dbops.Column(
name='key',
type='text',
),
dbops.Column(
name='bin',
type='bytea',
),
dbops.Column(
name='text',
type='text',
),
dbops.Column(
name='json',
type='jsonb',
),
],
constraints=[
dbops.PrimaryKey(
table_name=('edgedbinstdata', 'instdata'),
columns=['key'],
),
],
))
])
block = dbops.PLTopBlock()
commands.generate(block)
await _execute_block(ctx.conn, block)
mock_auth_nonce = scram.generate_nonce()
json_instance_data = {
'version': dict(buildmeta.get_version_dict()),
'catver': edbdef.EDGEDB_CATALOG_VERSION,
'mock_auth_nonce': mock_auth_nonce,
}
await _store_static_json_cache(
ctx,
'instancedata',
json.dumps(json_instance_data),
)
backend_params = ctx.cluster.get_runtime_params()
instance_params = backend_params.instance_params
await _store_static_json_cache(
ctx,
'backend_instance_params',
json.dumps(instance_params._asdict()),
)
if not backend_params.has_create_role:
json_single_role_metadata = {
'id': str(uuidgen.uuid1mc()),
'name': edbdef.EDGEDB_SUPERUSER,
'tenant_id': backend_params.tenant_id,
'builtin': False,
}
await _store_static_json_cache(
ctx,
'single_role_metadata',
json.dumps(json_single_role_metadata),
)
if not backend_params.has_create_database:
await _store_static_json_cache(
ctx,
f'{edbdef.EDGEDB_TEMPLATE_DB}metadata',
json.dumps({}),
)
await _store_static_json_cache(
ctx,
f'{edbdef.EDGEDB_SYSTEM_DB}metadata',
json.dumps({}),
)
return json_instance_data
async def bootstrap(cluster, args) -> bool:
pgconn = await cluster.connect()
pgconn.add_log_listener(_pg_log_listener)
std_schema = None
try:
membership = set()
session_user = cluster.get_connection_params().user
if session_user != edbdef.EDGEDB_SUPERUSER:
membership.add(session_user)
await _ensure_edgedb_role(
cluster,
pgconn,
edbdef.EDGEDB_SUPERUSER,
membership=membership,
is_superuser=True,
)
if session_user != edbdef.EDGEDB_SUPERUSER:
await _execute(
pgconn,
f'SET ROLE {edbdef.EDGEDB_SUPERUSER};',
)
cluster.set_default_session_authorization(edbdef.EDGEDB_SUPERUSER)
need_meta_bootstrap = await _ensure_edgedb_template_database(pgconn)
if need_meta_bootstrap:
conn = await cluster.connect(database=edbdef.EDGEDB_TEMPLATE_DB)
conn.add_log_listener(_pg_log_listener)
superuser_role = cluster.get_superuser_role()
if superuser_role:
block = dbops.PLTopBlock()
reassign = dbops.ReassignOwned(
session_user, edbdef.EDGEDB_SUPERUSER)
reassign.generate(block)
await _execute_block(conn, block)
try:
conn.add_log_listener(_pg_log_listener)
await _ensure_meta_schema(conn)
instancedata = await _populate_misc_instance_data(cluster)
std_schema = await _init_stdlib(
cluster, conn, testmode=args['testmode'])
await _bootstrap_config_spec(std_schema, cluster)
await _compile_sys_queries(std_schema, cluster)
schema = await _init_defaults(std_schema, std_schema, conn)
schema = await _populate_data(std_schema, schema, conn)
await _configure(std_schema, conn, cluster,
insecure=args['insecure'],
testmode=args['testmode'])
finally:
await conn.close()
await _ensure_edgedb_database(
pgconn,
edbdef.EDGEDB_SUPERUSER_DB,
edbdef.EDGEDB_SUPERUSER,
cluster=cluster)
else:
conn = await cluster.connect(database=edbdef.EDGEDB_SUPERUSER_DB)
try:
std_schema = await compiler.load_std_schema(conn)
config_spec = config.load_spec_from_schema(std_schema)
config.set_settings(config_spec)
instancedata = await _get_instance_data(conn)
finally:
await conn.close()
datadir_version = instancedata.get('version')
if datadir_version:
datadir_major = datadir_version.get('major')
expected_ver = buildmeta.get_version()
if datadir_major != expected_ver.major:
raise errors.ConfigurationError(
'database instance incompatible with this version of EdgeDB',
details=(
f'The database instance was initialized with '
f'EdgeDB version {datadir_major}, '
f'which is incompatible with this version '
f'{expected_ver.major}'
),
hint=(
f'You need to recreate the instance and upgrade '
f'using dump/restore.'
)
)
datadir_catver = instancedata.get('catver')
expected_catver = edbdef.EDGEDB_CATALOG_VERSION
if datadir_catver != expected_catver:
raise errors.ConfigurationError(
'database instance incompatible with this version of EdgeDB',
details=(
f'The database instance was initialized with '
f'EdgeDB format version {datadir_catver}, '
f'but this version of the server expects '
f'format version {expected_catver}'
),
hint=(
f'You need to recreate the instance and upgrade '
f'using dump/restore.'
)
)
await _ensure_edgedb_template_not_connectable(pgconn)
await _ensure_edgedb_role(
cluster, pgconn, args['default_database_user'], is_superuser=True)
await _execute(
pgconn,
f"SET ROLE {args['default_database_user']};",
)
await _ensure_edgedb_database(
pgconn,
args['default_database'],
args['default_database_user'],
cluster=cluster,
)
finally:
await pgconn.close()
return need_meta_bootstrap
async def _make_stdlib(testmode: bool, global_ids) -> StdlibBits:
schema = s_schema.Schema()
schema, _ = s_mod.Module.create_in_schema(schema, name='__derived__')
current_block = dbops.PLTopBlock()
std_texts = []
for modname in s_schema.STD_LIB + ('stdgraphql', ):
std_texts.append(s_std.get_std_module_text(modname))
if testmode:
std_texts.append(s_std.get_std_module_text('_testmode'))
ddl_text = '\n'.join(std_texts)
types: Set[uuid.UUID] = set()
std_plans: List[sd.Command] = []
for ddl_cmd in edgeql.parse_block(ddl_text):
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
modaliases={},
schema=schema,
stdmode=True)
if debug.flags.delta_plan_input:
debug.header('Delta Plan Input')
debug.dump(delta_command)
# Apply and adapt delta, build native delta plan, which
# will also update the schema.
schema, plan = _process_delta(delta_command, schema)
std_plans.append(delta_command)
types.update(plan.new_types)
plan.generate(current_block)
stdglobals = '\n'.join([
f'''CREATE SUPERUSER ROLE {edbdef.EDGEDB_SUPERUSER} {{
SET id := <uuid>'{global_ids[edbdef.EDGEDB_SUPERUSER]}'
}};''',
f'''CREATE DATABASE {edbdef.EDGEDB_TEMPLATE_DB} {{
SET id := <uuid>'{global_ids[edbdef.EDGEDB_TEMPLATE_DB]}'
}};''',
f'CREATE DATABASE {edbdef.EDGEDB_SUPERUSER_DB};',
])
context = sd.CommandContext(stdmode=True)
for ddl_cmd in edgeql.parse_block(stdglobals):
delta_command = s_ddl.delta_from_ddl(ddl_cmd,
modaliases={},
schema=schema,
stdmode=True)
schema = delta_command.apply(schema, context)
refldelta, classlayout, introparts = s_refl.generate_structure(schema)
reflschema, reflplan = _process_delta(refldelta, schema)
std_plans.append(refldelta)
assert current_block is not None
reflplan.generate(current_block)
subblock = current_block.add_block()
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=reflschema,
schema_class_layout=classlayout,
bootstrap_mode=True,
)
compilerctx = edbcompiler.new_compiler_context(reflschema)
for std_plan in std_plans:
compiler._compile_schema_storage_in_delta(
ctx=compilerctx,
delta=std_plan,
block=subblock,
is_internal_reflection=std_plan is refldelta,
stdmode=True,
)
sqltext = current_block.to_string()
compilerctx = edbcompiler.new_compiler_context(
reflschema,
schema_reflection_mode=True,
output_format=edbcompiler.IoFormat.JSON_ELEMENTS,
)
# The introspection query bits are returned in chunks
# because it's a large UNION and we currently generate SQL
# that is much harder for Posgres to plan as opposed to a
# straight flat UNION.
sql_introparts = []
for intropart in introparts:
introtokens = tokenizer.tokenize(intropart.encode())
units = compiler._compile(ctx=compilerctx, tokens=introtokens)
assert len(units) == 1 and len(units[0].sql) == 1
sql_intropart = units[0].sql[0].decode()
sql_introparts.append(sql_intropart)
introsql = ' UNION ALL '.join(sql_introparts)
return StdlibBits(
stdschema=schema,
reflschema=reflschema,
sqltext=sqltext,
types=types,
classlayout=classlayout,
introquery=introsql,
)
