def _compile_and_apply_ddl_command(self, ctx: CompileContext, cmd): current_tx = ctx.state.current_tx() schema = current_tx.get_schema() if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(cmd) # Do a dry-run on test_schema to canonicalize # the schema delta-commands. test_schema = schema context = self._new_delta_context(ctx) cmd.apply(test_schema, context=context) cmd.canonical = True # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = self._process_delta(ctx, cmd, schema) if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)): block = pg_dbops.SQLBlock() else: block = pg_dbops.PLTopBlock() plan.generate(block) sql = block.to_string().encode('utf-8') current_tx.update_schema(schema) if debug.flags.delta_execute: debug.header('Delta Script') debug.dump_code(sql, lexer='sql') return dbstate.DDLQuery(sql=(sql, ))
async def _populate_misc_instance_data(cluster, conn): commands = dbops.CommandGroup() commands.add_commands([ dbops.CreateSchema(name='edgedbinstdata'), ]) block = dbops.PLTopBlock() commands.generate(block) await _execute_block(conn, block) mock_auth_nonce = scram.generate_nonce() json_instance_data = { 'version': dict(buildmeta.get_version_dict()), 'catver': edbdef.EDGEDB_CATALOG_VERSION, 'mock_auth_nonce': mock_auth_nonce, } await _store_static_json_cache( cluster, 'instancedata', json.dumps(json_instance_data), ) return json_instance_data
async def _make_stdlib(testmode: bool): schema = s_schema.Schema() current_block = None std_texts = [] for modname in s_schema.STD_LIB + ('stdgraphql', ): std_texts.append(s_std.get_std_module_text(modname)) if testmode: std_texts.append(s_std.get_std_module_text('_testmode')) ddl_text = '\n'.join(std_texts) for ddl_cmd in edgeql.parse_block(ddl_text): delta_command = s_ddl.delta_from_ddl(ddl_cmd, schema=schema, modaliases={None: 'std'}, stdmode=True) if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(delta_command) # Do a dry-run on test_schema to canonicalize # the schema delta-commands. test_schema = schema context = sd.CommandContext() context.stdmode = True delta_command.apply(test_schema, context=context) # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = _process_delta(delta_command, schema) if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)): if (current_block is not None and not isinstance(current_block, dbops.SQLBlock)): raise errors.QueryError( 'cannot mix DATABASE commands with regular DDL ' 'commands in a single block') if current_block is None: current_block = dbops.SQLBlock() else: if (current_block is not None and not isinstance(current_block, dbops.PLTopBlock)): raise errors.QueryError( 'cannot mix DATABASE commands with regular DDL ' 'commands in a single block') if current_block is None: current_block = dbops.PLTopBlock() plan.generate(current_block) sql_text = current_block.to_string() return schema, sql_text
async def _populate_misc_instance_data(cluster, conn): commands = dbops.CommandGroup() commands.add_commands([ dbops.CreateSchema(name='edgedbinstdata'), dbops.CreateTable( dbops.Table( name=('edgedbinstdata', 'instdata'), columns=[ dbops.Column( name='key', type='text', ), dbops.Column( name='bin', type='bytea', ), dbops.Column( name='text', type='text', ), dbops.Column( name='json', type='jsonb', ), ], constraints=[ dbops.PrimaryKey( table_name=('edgedbinstdata', 'instdata'), columns=['key'], ), ], )) ]) block = dbops.PLTopBlock() commands.generate(block) await _execute_block(conn, block) mock_auth_nonce = scram.generate_nonce() json_instance_data = { 'version': dict(buildmeta.get_version_dict()), 'catver': edbdef.EDGEDB_CATALOG_VERSION, 'mock_auth_nonce': mock_auth_nonce, } await _store_static_json_cache( cluster, 'instancedata', json.dumps(json_instance_data), ) instance_params = cluster.get_runtime_params().instance_params await _store_static_json_cache( cluster, 'backend_instance_params', json.dumps(instance_params._asdict()), ) return json_instance_data
async def _amend_stdlib( ddl_text: str, stdlib: StdlibBits, ) -> Tuple[StdlibBits, str]: schema = stdlib.stdschema reflschema = stdlib.reflschema topblock = dbops.PLTopBlock() plans = [] context = sd.CommandContext() context.stdmode = True for ddl_cmd in edgeql.parse_block(ddl_text): assert isinstance(ddl_cmd, qlast.DDLCommand) delta_command = s_ddl.delta_from_ddl(ddl_cmd, modaliases={}, schema=schema, stdmode=True) if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(delta_command) # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = _process_delta(delta_command, schema) reflschema = delta_command.apply(reflschema, context) plan.generate(topblock) plans.append(plan) compiler = edbcompiler.new_compiler( std_schema=schema, reflection_schema=reflschema, schema_class_layout=stdlib.classlayout, ) compilerctx = edbcompiler.new_compiler_context( schema, bootstrap_mode=True, ) compilerctx = edbcompiler.new_compiler_context(schema) for plan in plans: compiler._compile_schema_storage_in_delta( ctx=compilerctx, delta=plan, block=topblock, ) sqltext = topblock.to_string() return stdlib._replace(stdschema=schema, reflschema=reflschema), sqltext
async def _ensure_edgedb_role( ctx: BootstrapContext, role_name: str, *, superuser: bool = False, builtin: bool = False, objid: Optional[uuid.UUID] = None, ) -> uuid.UUID: member_of = set() if superuser: member_of.add(edbdef.EDGEDB_SUPERGROUP) if objid is None: objid = uuidgen.uuid1mc() members = set() login_role = ctx.cluster.get_connection_params().user sup_role = ctx.cluster.get_role_name(edbdef.EDGEDB_SUPERUSER) if login_role != sup_role: members.add(login_role) instance_params = ctx.cluster.get_runtime_params().instance_params pg_role_name = ctx.cluster.get_role_name(role_name) role = dbops.Role( name=pg_role_name, superuser=(superuser and bool(instance_params.capabilities & pgcluster.BackendCapabilities.SUPERUSER_ACCESS)), allow_login=True, allow_createdb=True, allow_createrole=True, membership=[ctx.cluster.get_role_name(m) for m in member_of], members=members, metadata=dict( id=str(objid), name=role_name, tenant_id=instance_params.tenant_id, builtin=builtin, ), ) create_role = dbops.CreateRole( role, neg_conditions=[dbops.RoleExists(pg_role_name)], ) block = dbops.PLTopBlock() create_role.generate(block) await _execute_block(ctx.conn, block) return objid
async def _ensure_edgedb_role( cluster, conn, username, *, membership=(), is_superuser=False, builtin=False, objid=None, ) -> None: membership = set(membership) if is_superuser: superuser_role = cluster.get_superuser_role() if superuser_role: # If the cluster is exposing an explicit superuser role, # become a member of that instead of creating a superuser # role directly. membership.add(superuser_role) superuser_flag = False else: superuser_flag = True else: superuser_flag = False if objid is None: objid = uuidgen.uuid1mc() role = dbops.Role( name=username, is_superuser=superuser_flag, allow_login=True, allow_createdb=True, allow_createrole=True, membership=membership, metadata=dict( id=str(objid), builtin=builtin, ), ) create_role = dbops.CreateRole( role, neg_conditions=[dbops.RoleExists(username)], ) block = dbops.PLTopBlock() create_role.generate(block) await _execute_block(conn, block) return objid
async def _ensure_edgedb_role( cluster, conn, username, *, superuser=False, builtin=False, objid=None, ) -> None: member_of = set() if superuser: member_of.add(edbdef.EDGEDB_SUPERGROUP) if objid is None: objid = uuidgen.uuid1mc() members = set() login_role = cluster.get_connection_params().user if login_role != edbdef.EDGEDB_SUPERUSER: members.add(login_role) instance_params = cluster.get_runtime_params().instance_params role = dbops.Role( name=username, superuser=(superuser and bool(instance_params.capabilities & pgcluster.BackendCapabilities.SUPERUSER_ACCESS)), allow_login=True, allow_createdb=True, allow_createrole=True, membership=member_of, members=members, metadata=dict( id=str(objid), builtin=builtin, ), ) create_role = dbops.CreateRole( role, neg_conditions=[dbops.RoleExists(username)], ) block = dbops.PLTopBlock() create_role.generate(block) await _execute_block(conn, block) return objid
async def _configure(schema, conn, cluster, *, insecure=False, testmode=False): scripts = [] if not testmode: memory_kb = psutil.virtual_memory().total // 1024 settings = { 'shared_buffers': f'"{int(memory_kb * 0.2)}kB"', 'effective_cache_size': f'"{int(memory_kb * 0.5)}kB"', 'query_work_mem': f'"{6 * (2 ** 10)}kB"', } for setting, value in settings.items(): scripts.append(f''' CONFIGURE SYSTEM SET {setting} := {value}; ''') else: settings = {} if insecure: scripts.append(''' CONFIGURE SYSTEM INSERT Auth { priority := 0, method := (INSERT Trust), }; ''') config_spec = config.get_settings() for script in scripts: _, sql = compiler.compile_bootstrap_script( schema, schema, script, single_statement=True) if debug.flags.bootstrap: debug.header('Bootstrap') debug.dump_code(sql, lexer='sql') config_op_data = await conn.fetchval(sql) if config_op_data is not None and isinstance(config_op_data, str): config_op = config.Operation.from_json(config_op_data) settings = config_op.apply(config_spec, immutables.Map()) config_json = config.to_json(config_spec, settings) block = dbops.PLTopBlock() dbops.UpdateMetadata( dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB), {'sysconfig': json.loads(config_json)}, ).generate(block) await _execute_block(conn, block)
async def _configure( ctx: BootstrapContext, config_spec: config.Spec, schema: s_schema.Schema, compiler: edbcompiler.Compiler, ) -> None: settings: Mapping[str, config.SettingValue] = {} config_json = config.to_json(config_spec, settings, include_source=False) block = dbops.PLTopBlock() metadata = {'sysconfig': json.loads(config_json)} if ctx.cluster.get_runtime_params().has_create_database: dbops.UpdateMetadata( dbops.Database( name=ctx.cluster.get_db_name(edbdef.EDGEDB_SYSTEM_DB) ), metadata, ).generate(block) else: dbops.UpdateSingleDBMetadata( edbdef.EDGEDB_SYSTEM_DB, metadata, ).generate(block) await _execute_block(ctx.conn, block) backend_params = ctx.cluster.get_runtime_params() for setname in config_spec: setting = config_spec[setname] if ( setting.backend_setting and setting.default is not None and ( # Do not attempt to run CONFIGURE INSTANCE on # backends that don't support it. # TODO: this should be replaced by instance-wide # emulation at backend connection time. backend_params.has_configfile_access ) ): if isinstance(setting.default, statypes.Duration): val = f'<std::duration>"{setting.default.to_iso8601()}"' else: val = repr(setting.default) script = f''' CONFIGURE INSTANCE SET {setting.name} := {val}; ''' schema, sql = compile_bootstrap_script(compiler, schema, script) await _execute_ddl(ctx.conn, sql)
async def _configure( schema: s_schema.Schema, compiler: edbcompiler.Compiler, conn: asyncpg_con.Connection, cluster: pgcluster.BaseCluster, *, insecure: bool = False, ) -> None: config_spec = config.get_settings() scripts = [] settings: Mapping[str, config.SettingValue] = {} if insecure: scripts.append(''' CONFIGURE SYSTEM INSERT Auth { priority := 0, method := (INSERT Trust), }; ''') for script in scripts: _, sql = compile_bootstrap_script( compiler, schema, script, single_statement=True, ) if debug.flags.bootstrap: debug.header('Bootstrap') debug.dump_code(sql, lexer='sql') config_op_data = await conn.fetchval(sql) if config_op_data is not None and isinstance(config_op_data, str): config_op = config.Operation.from_json(config_op_data) settings = config_op.apply(config_spec, immutables.Map()) config_json = config.to_json(config_spec, settings, include_source=False) block = dbops.PLTopBlock() dbops.UpdateMetadata( dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB), { 'sysconfig': json.loads(config_json) }, ).generate(block) await _execute_block(conn, block)
async def _ensure_edgedb_user(conn, username, *, is_superuser=False): role = dbops.Role(name=username, is_superuser=is_superuser, allow_login=True, metadata=dict( id=str(uuidgen.uuid1mc()), __edgedb__='1', )) create_role = dbops.CreateRole(role=role, neg_conditions=[dbops.RoleExists(username)]) block = dbops.PLTopBlock() create_role.generate(block) await _execute_block(conn, block)
async def _ensure_edgedb_supergroup( ctx: BootstrapContext, role_name: str, *, member_of: Iterable[str] = (), members: Iterable[str] = (), ) -> None: member_of = set(member_of) instance_params = ctx.cluster.get_runtime_params().instance_params superuser_role = instance_params.base_superuser if superuser_role: # If the cluster is exposing an explicit superuser role, # become a member of that instead of creating a superuser # role directly. member_of.add(superuser_role) pg_role_name = ctx.cluster.get_role_name(role_name) role = dbops.Role( name=pg_role_name, superuser=bool(instance_params.capabilities & pgcluster.BackendCapabilities.SUPERUSER_ACCESS), allow_login=False, allow_createdb=True, allow_createrole=True, membership=member_of, members=members, ) create_role = dbops.CreateRole( role, neg_conditions=[dbops.RoleExists(pg_role_name)], ) block = dbops.PLTopBlock() create_role.generate(block) await _execute_block(ctx.conn, block)
async def _init_stdlib(cluster, conn, testmode, global_ids): in_dev_mode = devmode.is_in_dev_mode() specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO') if specified_cache_dir: cache_dir = pathlib.Path(specified_cache_dir) else: cache_dir = None stdlib_cache = 'backend-stdlib.pickle' tpldbdump_cache = 'backend-tpldbdump.sql' src_hash = buildmeta.hash_dirs( buildmeta.get_cache_src_dirs(), extra_files=[__file__], ) stdlib = buildmeta.read_data_cache(src_hash, stdlib_cache, source_dir=cache_dir) tpldbdump = buildmeta.read_data_cache(src_hash, tpldbdump_cache, source_dir=cache_dir, pickled=False) if stdlib is None: logger.info('Compiling the standard library...') stdlib = await _make_stdlib(in_dev_mode or testmode, global_ids) logger.info('Creating the necessary PostgreSQL extensions...') await metaschema.create_pg_extensions(conn) if tpldbdump is None: logger.info('Populating internal SQL structures...') await metaschema.bootstrap(conn) logger.info('Executing the standard library...') await _execute_ddl(conn, stdlib.sqltext) if in_dev_mode or specified_cache_dir: tpldbdump = cluster.dump_database( edbdef.EDGEDB_TEMPLATE_DB, exclude_schemas=['edgedbinstdata', 'edgedbext'], ) # Excluding the "edgedbext" schema above apparently # doesn't apply to extensions created in that schema, # so we have to resort to commenting out extension # statements in the dump. tpldbdump = re.sub( rb'^(CREATE|COMMENT ON) EXTENSION.*$', rb'-- \g<0>', tpldbdump, flags=re.MULTILINE, ) global_metadata = await conn.fetchval(f'''\ SELECT edgedb.shobj_metadata( (SELECT oid FROM pg_database WHERE datname = {ql(edbdef.EDGEDB_TEMPLATE_DB)}), 'pg_database' )''') pl_block = dbops.PLTopBlock() dbops.SetMetadata( dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB), json.loads(global_metadata), ).generate(pl_block) tpldbdump += b'\n' + pl_block.to_string().encode('utf-8') buildmeta.write_data_cache( tpldbdump, src_hash, tpldbdump_cache, pickled=False, target_dir=cache_dir, ) buildmeta.write_data_cache( stdlib, src_hash, stdlib_cache, target_dir=cache_dir, ) else: logger.info('Initializing the standard library...') await metaschema._execute_sql_script(conn, tpldbdump.decode('utf-8')) # When we restore a database from a dump, OIDs for non-system # Postgres types might get skewed as they are not part of the dump. # A good example of that is `std::bigint` which is implemented as # a custom domain type. The OIDs are stored under # `schema::Object.backend_id` property and are injected into # array query arguments. # # The code below re-syncs backend_id properties of EdgeDB builtin # types with the actual OIDs in the DB. compiler = edbcompiler.new_compiler( std_schema=stdlib.stdschema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) _, sql = compile_bootstrap_script( compiler, stdlib.reflschema, ''' UPDATE schema::ScalarType FILTER .builtin AND NOT (.abstract ?? False) SET { backend_id := sys::_get_pg_type_for_scalar_type(.id) } ''', expected_cardinality_one=False, single_statement=True, ) await conn.execute(sql) if not in_dev_mode and testmode: # Running tests on a production build. stdlib, testmode_sql = await _amend_stdlib( s_std.get_std_module_text('_testmode'), stdlib, ) await conn.execute(testmode_sql) await metaschema.generate_support_views( conn, stdlib.reflschema, ) # Make sure that schema backend_id properties are in sync with # the database. compiler = edbcompiler.new_compiler( std_schema=stdlib.stdschema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) _, sql = compile_bootstrap_script( compiler, stdlib.reflschema, ''' SELECT schema::ScalarType { id, backend_id, } FILTER .builtin AND NOT (.abstract ?? False); ''', expected_cardinality_one=False, single_statement=True, ) schema = stdlib.stdschema typemap = await conn.fetchval(sql) for entry in json.loads(typemap): t = schema.get_by_id(uuidgen.UUID(entry['id'])) schema = t.set_field_value(schema, 'backend_id', entry['backend_id']) stdlib = stdlib._replace(stdschema=schema) await _store_static_bin_cache( cluster, 'stdschema', pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( cluster, 'reflschema', pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( cluster, 'global_schema', pickle.dumps(stdlib.global_schema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( cluster, 'classlayout', pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_text_cache( cluster, 'local_intro_query', stdlib.local_intro_query, ) await _store_static_text_cache( cluster, 'global_intro_query', stdlib.global_intro_query, ) await metaschema.generate_support_views(conn, stdlib.reflschema) await metaschema.generate_support_functions(conn, stdlib.reflschema) compiler = edbcompiler.new_compiler( std_schema=schema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) await metaschema.generate_more_support_functions(conn, compiler, stdlib.reflschema, testmode) return stdlib, compiler
async def _make_stdlib(testmode: bool, global_ids) -> StdlibBits: schema = s_schema.ChainedSchema( s_schema.FlatSchema(), s_schema.FlatSchema(), s_schema.FlatSchema(), ) schema, _ = s_mod.Module.create_in_schema( schema, name=sn.UnqualName('__derived__'), ) current_block = dbops.PLTopBlock() std_texts = [] for modname in s_schema.STD_SOURCES: std_texts.append(s_std.get_std_module_text(modname)) if testmode: std_texts.append(s_std.get_std_module_text(sn.UnqualName('_testmode'))) ddl_text = '\n'.join(std_texts) types: Set[uuid.UUID] = set() std_plans: List[sd.Command] = [] for ddl_cmd in edgeql.parse_block(ddl_text): assert isinstance(ddl_cmd, qlast.DDLCommand) delta_command = s_ddl.delta_from_ddl(ddl_cmd, modaliases={}, schema=schema, stdmode=True) if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(delta_command) # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = _process_delta(delta_command, schema) std_plans.append(delta_command) types.update(plan.new_types) plan.generate(current_block) _, schema_version = s_std.make_schema_version(schema) schema, plan = _process_delta(schema_version, schema) std_plans.append(schema_version) plan.generate(current_block) stdglobals = '\n'.join([ f'''CREATE SUPERUSER ROLE {edbdef.EDGEDB_SUPERUSER} {{ SET id := <uuid>'{global_ids[edbdef.EDGEDB_SUPERUSER]}' }};''', ]) schema = await _execute_edgeql_ddl(schema, stdglobals) _, global_schema_version = s_std.make_global_schema_version(schema) schema, plan = _process_delta(global_schema_version, schema) std_plans.append(global_schema_version) plan.generate(current_block) reflection = s_refl.generate_structure(schema) reflschema, reflplan = _process_delta(reflection.intro_schema_delta, schema) assert current_block is not None reflplan.generate(current_block) subblock = current_block.add_block() compiler = edbcompiler.new_compiler( std_schema=schema.get_top_schema(), reflection_schema=reflschema.get_top_schema(), schema_class_layout=reflection.class_layout, # type: ignore ) compilerctx = edbcompiler.new_compiler_context( user_schema=reflschema.get_top_schema(), global_schema=schema.get_global_schema(), bootstrap_mode=True, ) for std_plan in std_plans: compiler._compile_schema_storage_in_delta( ctx=compilerctx, delta=std_plan, block=subblock, ) compilerctx = edbcompiler.new_compiler_context( user_schema=reflschema.get_top_schema(), global_schema=schema.get_global_schema(), bootstrap_mode=True, internal_schema_mode=True, ) compiler._compile_schema_storage_in_delta( ctx=compilerctx, delta=reflection.intro_schema_delta, block=subblock, ) sqltext = current_block.to_string() compilerctx = edbcompiler.new_compiler_context( user_schema=reflschema.get_top_schema(), global_schema=schema.get_global_schema(), schema_reflection_mode=True, output_format=edbcompiler.IoFormat.JSON_ELEMENTS, ) # The introspection query bits are returned in chunks # because it's a large UNION and we currently generate SQL # that is much harder for Posgres to plan as opposed to a # straight flat UNION. sql_intro_local_parts = [] sql_intro_global_parts = [] for intropart in reflection.local_intro_parts: sql_intro_local_parts.append( compile_single_query( intropart, compiler=compiler, compilerctx=compilerctx, ), ) for intropart in reflection.global_intro_parts: sql_intro_global_parts.append( compile_single_query( intropart, compiler=compiler, compilerctx=compilerctx, ), ) local_intro_sql = ' UNION ALL '.join(sql_intro_local_parts) local_intro_sql = f''' WITH intro(c) AS ({local_intro_sql}) SELECT json_agg(intro.c) FROM intro ''' global_intro_sql = ' UNION ALL '.join(sql_intro_global_parts) global_intro_sql = f''' WITH intro(c) AS ({global_intro_sql}) SELECT json_agg(intro.c) FROM intro ''' return StdlibBits( stdschema=schema.get_top_schema(), reflschema=reflschema.get_top_schema(), global_schema=schema.get_global_schema(), sqltext=sqltext, types=types, classlayout=reflection.class_layout, local_intro_query=local_intro_sql, global_intro_query=global_intro_sql, )
async def _init_stdlib( ctx: BootstrapContext, testmode: bool, global_ids: Mapping[str, uuid.UUID], ) -> Tuple[StdlibBits, config.Spec, edbcompiler.Compiler]: in_dev_mode = devmode.is_in_dev_mode() conn = ctx.conn cluster = ctx.cluster specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO') if not specified_cache_dir: cache_dir = None else: cache_dir = pathlib.Path(specified_cache_dir) stdlib_cache = f'backend-stdlib.pickle' tpldbdump_cache = f'backend-tpldbdump.sql' src_hash = buildmeta.hash_dirs( buildmeta.get_cache_src_dirs(), extra_files=[__file__], ) stdlib = buildmeta.read_data_cache( src_hash, stdlib_cache, source_dir=cache_dir) tpldbdump = buildmeta.read_data_cache( src_hash, tpldbdump_cache, source_dir=cache_dir, pickled=False) if stdlib is None: logger.info('Compiling the standard library...') stdlib = await _make_stdlib(ctx, in_dev_mode or testmode, global_ids) logger.info('Creating the necessary PostgreSQL extensions...') await metaschema.create_pg_extensions(conn) config_spec = config.load_spec_from_schema(stdlib.stdschema) config.set_settings(config_spec) if tpldbdump is None: logger.info('Populating internal SQL structures...') await metaschema.bootstrap(conn, config_spec) logger.info('Executing the standard library...') await _execute_ddl(conn, stdlib.sqltext) if in_dev_mode or specified_cache_dir: tpl_db_name = edbdef.EDGEDB_TEMPLATE_DB tpl_pg_db_name = cluster.get_db_name(tpl_db_name) tpl_pg_db_name_dyn = ( f"edgedb.get_database_backend_name({ql(tpl_db_name)})") tpldbdump = await cluster.dump_database( tpl_pg_db_name, exclude_schemas=['edgedbinstdata', 'edgedbext'], dump_object_owners=False, ) # Excluding the "edgedbext" schema above apparently # doesn't apply to extensions created in that schema, # so we have to resort to commenting out extension # statements in the dump. tpldbdump = re.sub( rb'^(CREATE|COMMENT ON) EXTENSION.*$', rb'-- \g<0>', tpldbdump, flags=re.MULTILINE, ) global_metadata = await conn.fetchval( f'SELECT edgedb.get_database_metadata({ql(tpl_db_name)})', ) global_metadata = json.loads(global_metadata) pl_block = dbops.PLTopBlock() set_metadata_text = dbops.SetMetadata( dbops.Database(name='__dummy_placeholder_database__'), global_metadata, ).code(pl_block) set_metadata_text = set_metadata_text.replace( '__dummy_placeholder_database__', f"' || quote_ident({tpl_pg_db_name_dyn}) || '", ) set_single_db_metadata_text = dbops.SetSingleDBMetadata( edbdef.EDGEDB_TEMPLATE_DB, global_metadata ).code(pl_block) pl_block.add_command(textwrap.dedent(f"""\ IF (edgedb.get_backend_capabilities() & {int(params.BackendCapabilities.CREATE_DATABASE)}) != 0 THEN {textwrap.indent(set_metadata_text, ' ')} ELSE {textwrap.indent(set_single_db_metadata_text, ' ')} END IF """)) text = pl_block.to_string() tpldbdump += b'\n' + text.encode('utf-8') buildmeta.write_data_cache( tpldbdump, src_hash, tpldbdump_cache, pickled=False, target_dir=cache_dir, ) buildmeta.write_data_cache( stdlib, src_hash, stdlib_cache, target_dir=cache_dir, ) else: logger.info('Initializing the standard library...') await metaschema._execute_sql_script(conn, tpldbdump.decode('utf-8')) # Restore the search_path as the dump might have altered it. await conn.execute( "SELECT pg_catalog.set_config('search_path', 'edgedb', false)") if not in_dev_mode and testmode: # Running tests on a production build. stdlib, testmode_sql = await _amend_stdlib( ctx, s_std.get_std_module_text(sn.UnqualName('_testmode')), stdlib, ) await conn.execute(testmode_sql) # _testmode includes extra config settings, so make sure # those are picked up. config_spec = config.load_spec_from_schema(stdlib.stdschema) config.set_settings(config_spec) # Make sure that schema backend_id properties are in sync with # the database. compiler = edbcompiler.new_compiler( std_schema=stdlib.stdschema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) _, sql = compile_bootstrap_script( compiler, stdlib.reflschema, ''' SELECT schema::ScalarType { id, backend_id, } FILTER .builtin AND NOT (.abstract ?? False); ''', expected_cardinality_one=False, single_statement=True, ) schema = stdlib.stdschema typemap = await conn.fetchval(sql) for entry in json.loads(typemap): t = schema.get_by_id(uuidgen.UUID(entry['id'])) schema = t.set_field_value( schema, 'backend_id', entry['backend_id']) stdlib = stdlib._replace(stdschema=schema) await _store_static_bin_cache( ctx, 'stdschema', pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( ctx, 'reflschema', pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( ctx, 'global_schema', pickle.dumps(stdlib.global_schema, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_bin_cache( ctx, 'classlayout', pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL), ) await _store_static_text_cache( ctx, 'local_intro_query', stdlib.local_intro_query, ) await _store_static_text_cache( ctx, 'global_intro_query', stdlib.global_intro_query, ) await metaschema.generate_support_views( conn, stdlib.reflschema, cluster.get_runtime_params() ) await metaschema.generate_support_functions(conn, stdlib.reflschema) compiler = edbcompiler.new_compiler( std_schema=schema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) await metaschema.generate_more_support_functions( conn, compiler, stdlib.reflschema, testmode) if tpldbdump is not None: # When we restore a database from a dump, OIDs for non-system # Postgres types might get skewed as they are not part of the dump. # A good example of that is `std::bigint` which is implemented as # a custom domain type. The OIDs are stored under # `schema::Object.backend_id` property and are injected into # array query arguments. # # The code below re-syncs backend_id properties of EdgeDB builtin # types with the actual OIDs in the DB. compiler = edbcompiler.new_compiler( std_schema=stdlib.stdschema, reflection_schema=stdlib.reflschema, schema_class_layout=stdlib.classlayout, ) _, sql = compile_bootstrap_script( compiler, stdlib.reflschema, ''' UPDATE schema::Type FILTER .builtin AND NOT (.abstract ?? False) AND schema::Type IS schema::ScalarType | schema::Tuple SET { backend_id := sys::_get_pg_type_for_edgedb_type( .id, <uuid>{} ) } ''', expected_cardinality_one=False, single_statement=True, ) await conn.execute(sql) _, sql = compile_bootstrap_script( compiler, stdlib.reflschema, ''' UPDATE schema::Array FILTER .builtin AND NOT (.abstract ?? False) SET { backend_id := sys::_get_pg_type_for_edgedb_type( .id, .element_type.id, ) } ''', expected_cardinality_one=False, single_statement=True, ) await conn.execute(sql) await _store_static_json_cache( ctx, 'configspec', config.spec_to_json(config_spec), ) return stdlib, config_spec, compiler
async def _make_stdlib( testmode: bool) -> Tuple[s_schema.Schema, str, Set[uuid.UUID]]: schema = s_schema.Schema() schema, _ = s_mod.Module.create_in_schema(schema, name='__derived__') schema = s_pseudo.populate_types(schema) current_block = None std_texts = [] for modname in s_schema.STD_LIB + ('stdgraphql', ): std_texts.append(s_std.get_std_module_text(modname)) if testmode: std_texts.append(s_std.get_std_module_text('_testmode')) ddl_text = '\n'.join(std_texts) new_types: Set[uuid.UUID] = set() for ddl_cmd in edgeql.parse_block(ddl_text): delta_command = s_ddl.delta_from_ddl(ddl_cmd, modaliases={}, schema=schema, stdmode=True) if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(delta_command) # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = _process_delta(delta_command, schema) if isinstance(plan, (s_db.CreateDatabase, s_db.DropDatabase)): if (current_block is not None and not isinstance(current_block, dbops.SQLBlock)): raise errors.QueryError( 'cannot mix DATABASE commands with regular DDL ' 'commands in a single block') if current_block is None: current_block = dbops.SQLBlock() else: new_types.update(plan.new_types) if (current_block is not None and not isinstance(current_block, dbops.PLTopBlock)): raise errors.QueryError( 'cannot mix DATABASE commands with regular DDL ' 'commands in a single block') if current_block is None: current_block = dbops.PLTopBlock() plan.generate(current_block) assert current_block is not None sql_text = current_block.to_string() mods = { mod.get_name(schema) for mod in schema.get_modules() if mod.get_builtin(schema) } if mods != s_schema.STD_MODULES: raise errors.SchemaError( f'modules {s_schema.STD_MODULES - mods} are not marked as builtin') return schema, sql_text, new_types
async def _populate_misc_instance_data( ctx: BootstrapContext, ) -> Dict[str, Any]: commands = dbops.CommandGroup() commands.add_commands([ dbops.CreateSchema(name='edgedbinstdata'), dbops.CreateTable(dbops.Table( name=('edgedbinstdata', 'instdata'), columns=[ dbops.Column( name='key', type='text', ), dbops.Column( name='bin', type='bytea', ), dbops.Column( name='text', type='text', ), dbops.Column( name='json', type='jsonb', ), ], constraints=[ dbops.PrimaryKey( table_name=('edgedbinstdata', 'instdata'), columns=['key'], ), ], )) ]) block = dbops.PLTopBlock() commands.generate(block) await _execute_block(ctx.conn, block) mock_auth_nonce = scram.generate_nonce() json_instance_data = { 'version': dict(buildmeta.get_version_dict()), 'catver': edbdef.EDGEDB_CATALOG_VERSION, 'mock_auth_nonce': mock_auth_nonce, } await _store_static_json_cache( ctx, 'instancedata', json.dumps(json_instance_data), ) backend_params = ctx.cluster.get_runtime_params() instance_params = backend_params.instance_params await _store_static_json_cache( ctx, 'backend_instance_params', json.dumps(instance_params._asdict()), ) if not backend_params.has_create_role: json_single_role_metadata = { 'id': str(uuidgen.uuid1mc()), 'name': edbdef.EDGEDB_SUPERUSER, 'tenant_id': backend_params.tenant_id, 'builtin': False, } await _store_static_json_cache( ctx, 'single_role_metadata', json.dumps(json_single_role_metadata), ) if not backend_params.has_create_database: await _store_static_json_cache( ctx, f'{edbdef.EDGEDB_TEMPLATE_DB}metadata', json.dumps({}), ) await _store_static_json_cache( ctx, f'{edbdef.EDGEDB_SYSTEM_DB}metadata', json.dumps({}), ) return json_instance_data
async def bootstrap(cluster, args) -> bool: pgconn = await cluster.connect() pgconn.add_log_listener(_pg_log_listener) std_schema = None try: membership = set() session_user = cluster.get_connection_params().user if session_user != edbdef.EDGEDB_SUPERUSER: membership.add(session_user) await _ensure_edgedb_role( cluster, pgconn, edbdef.EDGEDB_SUPERUSER, membership=membership, is_superuser=True, ) if session_user != edbdef.EDGEDB_SUPERUSER: await _execute( pgconn, f'SET ROLE {edbdef.EDGEDB_SUPERUSER};', ) cluster.set_default_session_authorization(edbdef.EDGEDB_SUPERUSER) need_meta_bootstrap = await _ensure_edgedb_template_database(pgconn) if need_meta_bootstrap: conn = await cluster.connect(database=edbdef.EDGEDB_TEMPLATE_DB) conn.add_log_listener(_pg_log_listener) superuser_role = cluster.get_superuser_role() if superuser_role: block = dbops.PLTopBlock() reassign = dbops.ReassignOwned( session_user, edbdef.EDGEDB_SUPERUSER) reassign.generate(block) await _execute_block(conn, block) try: conn.add_log_listener(_pg_log_listener) await _ensure_meta_schema(conn) instancedata = await _populate_misc_instance_data(cluster) std_schema = await _init_stdlib( cluster, conn, testmode=args['testmode']) await _bootstrap_config_spec(std_schema, cluster) await _compile_sys_queries(std_schema, cluster) schema = await _init_defaults(std_schema, std_schema, conn) schema = await _populate_data(std_schema, schema, conn) await _configure(std_schema, conn, cluster, insecure=args['insecure'], testmode=args['testmode']) finally: await conn.close() await _ensure_edgedb_database( pgconn, edbdef.EDGEDB_SUPERUSER_DB, edbdef.EDGEDB_SUPERUSER, cluster=cluster) else: conn = await cluster.connect(database=edbdef.EDGEDB_SUPERUSER_DB) try: std_schema = await compiler.load_std_schema(conn) config_spec = config.load_spec_from_schema(std_schema) config.set_settings(config_spec) instancedata = await _get_instance_data(conn) finally: await conn.close() datadir_version = instancedata.get('version') if datadir_version: datadir_major = datadir_version.get('major') expected_ver = buildmeta.get_version() if datadir_major != expected_ver.major: raise errors.ConfigurationError( 'database instance incompatible with this version of EdgeDB', details=( f'The database instance was initialized with ' f'EdgeDB version {datadir_major}, ' f'which is incompatible with this version ' f'{expected_ver.major}' ), hint=( f'You need to recreate the instance and upgrade ' f'using dump/restore.' ) ) datadir_catver = instancedata.get('catver') expected_catver = edbdef.EDGEDB_CATALOG_VERSION if datadir_catver != expected_catver: raise errors.ConfigurationError( 'database instance incompatible with this version of EdgeDB', details=( f'The database instance was initialized with ' f'EdgeDB format version {datadir_catver}, ' f'but this version of the server expects ' f'format version {expected_catver}' ), hint=( f'You need to recreate the instance and upgrade ' f'using dump/restore.' ) ) await _ensure_edgedb_template_not_connectable(pgconn) await _ensure_edgedb_role( cluster, pgconn, args['default_database_user'], is_superuser=True) await _execute( pgconn, f"SET ROLE {args['default_database_user']};", ) await _ensure_edgedb_database( pgconn, args['default_database'], args['default_database_user'], cluster=cluster, ) finally: await pgconn.close() return need_meta_bootstrap
async def _make_stdlib(testmode: bool, global_ids) -> StdlibBits: schema = s_schema.Schema() schema, _ = s_mod.Module.create_in_schema(schema, name='__derived__') current_block = dbops.PLTopBlock() std_texts = [] for modname in s_schema.STD_LIB + ('stdgraphql', ): std_texts.append(s_std.get_std_module_text(modname)) if testmode: std_texts.append(s_std.get_std_module_text('_testmode')) ddl_text = '\n'.join(std_texts) types: Set[uuid.UUID] = set() std_plans: List[sd.Command] = [] for ddl_cmd in edgeql.parse_block(ddl_text): delta_command = s_ddl.delta_from_ddl(ddl_cmd, modaliases={}, schema=schema, stdmode=True) if debug.flags.delta_plan_input: debug.header('Delta Plan Input') debug.dump(delta_command) # Apply and adapt delta, build native delta plan, which # will also update the schema. schema, plan = _process_delta(delta_command, schema) std_plans.append(delta_command) types.update(plan.new_types) plan.generate(current_block) stdglobals = '\n'.join([ f'''CREATE SUPERUSER ROLE {edbdef.EDGEDB_SUPERUSER} {{ SET id := <uuid>'{global_ids[edbdef.EDGEDB_SUPERUSER]}' }};''', f'''CREATE DATABASE {edbdef.EDGEDB_TEMPLATE_DB} {{ SET id := <uuid>'{global_ids[edbdef.EDGEDB_TEMPLATE_DB]}' }};''', f'CREATE DATABASE {edbdef.EDGEDB_SUPERUSER_DB};', ]) context = sd.CommandContext(stdmode=True) for ddl_cmd in edgeql.parse_block(stdglobals): delta_command = s_ddl.delta_from_ddl(ddl_cmd, modaliases={}, schema=schema, stdmode=True) schema = delta_command.apply(schema, context) refldelta, classlayout, introparts = s_refl.generate_structure(schema) reflschema, reflplan = _process_delta(refldelta, schema) std_plans.append(refldelta) assert current_block is not None reflplan.generate(current_block) subblock = current_block.add_block() compiler = edbcompiler.new_compiler( std_schema=schema, reflection_schema=reflschema, schema_class_layout=classlayout, bootstrap_mode=True, ) compilerctx = edbcompiler.new_compiler_context(reflschema) for std_plan in std_plans: compiler._compile_schema_storage_in_delta( ctx=compilerctx, delta=std_plan, block=subblock, is_internal_reflection=std_plan is refldelta, stdmode=True, ) sqltext = current_block.to_string() compilerctx = edbcompiler.new_compiler_context( reflschema, schema_reflection_mode=True, output_format=edbcompiler.IoFormat.JSON_ELEMENTS, ) # The introspection query bits are returned in chunks # because it's a large UNION and we currently generate SQL # that is much harder for Posgres to plan as opposed to a # straight flat UNION. sql_introparts = [] for intropart in introparts: introtokens = tokenizer.tokenize(intropart.encode()) units = compiler._compile(ctx=compilerctx, tokens=introtokens) assert len(units) == 1 and len(units[0].sql) == 1 sql_intropart = units[0].sql[0].decode() sql_introparts.append(sql_intropart) introsql = ' UNION ALL '.join(sql_introparts) return StdlibBits( stdschema=schema, reflschema=reflschema, sqltext=sqltext, types=types, classlayout=classlayout, introquery=introsql, )