def run(self):
import edb as _edb
from edb.server.buildmeta import hash_dirs, get_cache_src_dirs
build = self.get_finalized_command('build')
pkg_dir = pathlib.Path(_edb.__path__[0])
if self.type == 'parsers':
parser_hash = hash_dirs(
[(pkg_dir / 'edgeql/parser/grammar', '.py')],
extra_files=[pkg_dir / 'edgeql-parser/src/keywords.rs'],
)
print(binascii.hexlify(parser_hash).decode())
elif self.type == 'postgres':
print(_get_pg_source_stamp().strip())
elif self.type == 'bootstrap':
bootstrap_hash = hash_dirs(
get_cache_src_dirs(),
extra_files=[pkg_dir / 'server/bootstrap.py'],
)
print(binascii.hexlify(bootstrap_hash).decode())
elif self.type == 'rust':
rust_hash = hash_dirs([
(pkg_dir / 'edgeql-parser', '.rs'),
(pkg_dir / 'edgeql-rust', '.rs'),
(pkg_dir / 'graphql-rewrite', '.rs'),
],
extra_files=[
pkg_dir / 'edgeql-parser/Cargo.toml',
pkg_dir / 'edgeql-rust/Cargo.toml',
pkg_dir / 'graphql-rewrite/Cargo.toml',
])
print(binascii.hexlify(rust_hash).decode())
elif self.type == 'ext':
ext_hash = hash_dirs([
(pkg_dir, '.pyx'),
(pkg_dir, '.pyi'),
(pkg_dir, '.pxd'),
(pkg_dir, '.pxi'),
])
print(binascii.hexlify(ext_hash).decode())
elif self.type == 'cli':
print(_get_edgedbcli_rev())
elif self.type == 'build_temp':
print(pathlib.Path(build.build_temp).resolve())
elif self.type == 'build_lib':
print(pathlib.Path(build.build_lib).resolve())
else:
raise RuntimeError(f'Illegal --type={self.type}; can only be: '
'cli, rust, ext, postgres, bootstrap, parsers,'
'build_temp or build_lib')
def _load_reflection_schema():
global _refl_schema
global _schema_class_layout
if _refl_schema is None:
std_dirs_hash = buildmeta.hash_dirs(s_std.CACHE_SRC_DIRS)
cache = None
if devmode.is_in_dev_mode():
cache = buildmeta.read_data_cache(std_dirs_hash,
'transient-reflschema.pickle')
if cache is not None:
reflschema, classlayout = cache
else:
std_schema = _load_std_schema()
reflection = s_refl.generate_structure(std_schema)
classlayout = reflection.class_layout
context = sd.CommandContext()
context.stdmode = True
reflschema = reflection.intro_schema_delta.apply(
std_schema, context)
if devmode.is_in_dev_mode():
buildmeta.write_data_cache(
(reflschema, classlayout),
std_dirs_hash,
'transient-reflschema.pickle',
)
_refl_schema = reflschema
_schema_class_layout = classlayout
return _refl_schema, _schema_class_layout
def run(self, *args, **kwargs):
import edb as _edb
from edb.server.buildmeta import hash_dirs
parser_hash = hash_dirs([(
os.path.join(_edb.__path__[0], 'edgeql/parser/grammar'),
'.py')])
proc = subprocess.run(
['git', 'submodule', 'status', 'postgres'],
stdout=subprocess.PIPE, universal_newlines=True, check=True)
postgres_revision, _, _ = proc.stdout[1:].partition(' ')
print(f'{binascii.hexlify(parser_hash).decode()}-{postgres_revision}')
def _load_std_schema():
global _std_schema
if _std_schema is None:
std_dirs_hash = buildmeta.hash_dirs(s_std.CACHE_SRC_DIRS)
schema = None
if devmode.is_in_dev_mode():
schema = buildmeta.read_data_cache(std_dirs_hash,
'transient-stdschema.pickle')
if schema is None:
schema = s_schema.FlatSchema()
for modname in s_schema.STD_LIB + ('stdgraphql', ):
schema = s_std.load_std_module(schema, modname)
if devmode.is_in_dev_mode():
buildmeta.write_data_cache(schema, std_dirs_hash,
'transient-stdschema.pickle')
_std_schema = schema
return _std_schema
async def _init_stdlib(cluster, conn, testmode, global_ids):
in_dev_mode = devmode.is_in_dev_mode()
specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO')
if specified_cache_dir:
cache_dir = pathlib.Path(specified_cache_dir)
else:
cache_dir = None
stdlib_cache = 'backend-stdlib.pickle'
tpldbdump_cache = 'backend-tpldbdump.sql'
src_hash = buildmeta.hash_dirs(
buildmeta.get_cache_src_dirs(),
extra_files=[__file__],
)
stdlib = buildmeta.read_data_cache(src_hash,
stdlib_cache,
source_dir=cache_dir)
tpldbdump = buildmeta.read_data_cache(src_hash,
tpldbdump_cache,
source_dir=cache_dir,
pickled=False)
if stdlib is None:
logger.info('Compiling the standard library...')
stdlib = await _make_stdlib(in_dev_mode or testmode, global_ids)
logger.info('Creating the necessary PostgreSQL extensions...')
await metaschema.create_pg_extensions(conn)
if tpldbdump is None:
logger.info('Populating internal SQL structures...')
await metaschema.bootstrap(conn)
logger.info('Executing the standard library...')
await _execute_ddl(conn, stdlib.sqltext)
if in_dev_mode or specified_cache_dir:
tpldbdump = cluster.dump_database(
edbdef.EDGEDB_TEMPLATE_DB,
exclude_schemas=['edgedbinstdata', 'edgedbext'],
)
# Excluding the "edgedbext" schema above apparently
# doesn't apply to extensions created in that schema,
# so we have to resort to commenting out extension
# statements in the dump.
tpldbdump = re.sub(
rb'^(CREATE|COMMENT ON) EXTENSION.*$',
rb'-- \g<0>',
tpldbdump,
flags=re.MULTILINE,
)
global_metadata = await conn.fetchval(f'''\
SELECT edgedb.shobj_metadata(
(SELECT oid FROM pg_database
WHERE datname = {ql(edbdef.EDGEDB_TEMPLATE_DB)}),
'pg_database'
)''')
pl_block = dbops.PLTopBlock()
dbops.SetMetadata(
dbops.Database(name=edbdef.EDGEDB_TEMPLATE_DB),
json.loads(global_metadata),
).generate(pl_block)
tpldbdump += b'\n' + pl_block.to_string().encode('utf-8')
buildmeta.write_data_cache(
tpldbdump,
src_hash,
tpldbdump_cache,
pickled=False,
target_dir=cache_dir,
)
buildmeta.write_data_cache(
stdlib,
src_hash,
stdlib_cache,
target_dir=cache_dir,
)
else:
logger.info('Initializing the standard library...')
await metaschema._execute_sql_script(conn, tpldbdump.decode('utf-8'))
# When we restore a database from a dump, OIDs for non-system
# Postgres types might get skewed as they are not part of the dump.
# A good example of that is `std::bigint` which is implemented as
# a custom domain type. The OIDs are stored under
# `schema::Object.backend_id` property and are injected into
# array query arguments.
#
# The code below re-syncs backend_id properties of EdgeDB builtin
# types with the actual OIDs in the DB.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
UPDATE schema::ScalarType
FILTER .builtin AND NOT (.abstract ?? False)
SET {
backend_id := sys::_get_pg_type_for_scalar_type(.id)
}
''',
expected_cardinality_one=False,
single_statement=True,
)
await conn.execute(sql)
if not in_dev_mode and testmode:
# Running tests on a production build.
stdlib, testmode_sql = await _amend_stdlib(
s_std.get_std_module_text('_testmode'),
stdlib,
)
await conn.execute(testmode_sql)
await metaschema.generate_support_views(
conn,
stdlib.reflschema,
)
# Make sure that schema backend_id properties are in sync with
# the database.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
SELECT schema::ScalarType {
id,
backend_id,
} FILTER .builtin AND NOT (.abstract ?? False);
''',
expected_cardinality_one=False,
single_statement=True,
)
schema = stdlib.stdschema
typemap = await conn.fetchval(sql)
for entry in json.loads(typemap):
t = schema.get_by_id(uuidgen.UUID(entry['id']))
schema = t.set_field_value(schema, 'backend_id', entry['backend_id'])
stdlib = stdlib._replace(stdschema=schema)
await _store_static_bin_cache(
cluster,
'stdschema',
pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'reflschema',
pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'global_schema',
pickle.dumps(stdlib.global_schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'classlayout',
pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_text_cache(
cluster,
'local_intro_query',
stdlib.local_intro_query,
)
await _store_static_text_cache(
cluster,
'global_intro_query',
stdlib.global_intro_query,
)
await metaschema.generate_support_views(conn, stdlib.reflschema)
await metaschema.generate_support_functions(conn, stdlib.reflschema)
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
)
await metaschema.generate_more_support_functions(conn, compiler,
stdlib.reflschema,
testmode)
return stdlib, compiler
async def _init_stdlib(cluster, conn, testmode, global_ids):
in_dev_mode = devmode.is_in_dev_mode()
specified_cache_dir = os.environ.get('_EDGEDB_WRITE_DATA_CACHE_TO')
if specified_cache_dir:
cache_dir = pathlib.Path(specified_cache_dir)
else:
cache_dir = None
stdlib_cache = 'backend-stdlib.pickle'
tpldbdump_cache = 'backend-tpldbdump.sql'
src_hash = buildmeta.hash_dirs(CACHE_SRC_DIRS)
stdlib = buildmeta.read_data_cache(src_hash,
stdlib_cache,
source_dir=cache_dir)
tpldbdump = buildmeta.read_data_cache(src_hash,
tpldbdump_cache,
source_dir=cache_dir,
pickled=False)
if stdlib is None:
stdlib = await _make_stdlib(in_dev_mode or testmode, global_ids)
cache_hit = False
else:
cache_hit = True
if tpldbdump is None:
await _ensure_meta_schema(conn)
await _execute_ddl(conn, stdlib.sqltext)
if in_dev_mode or specified_cache_dir:
tpldbdump = cluster.dump_database(edbdef.EDGEDB_TEMPLATE_DB,
exclude_schema='edgedbinstdata')
buildmeta.write_data_cache(
tpldbdump,
src_hash,
tpldbdump_cache,
pickled=False,
target_dir=cache_dir,
)
else:
cluster.restore_database(edbdef.EDGEDB_TEMPLATE_DB, tpldbdump)
# When we restore a database from a dump, OIDs for non-system
# Postgres types might get skewed as they are not part of the dump.
# A good example of that is `std::bigint` which is implemented as
# a custom domain type. The OIDs are stored under
# `schema::Object.backend_id` property and are injected into
# array query arguments.
#
# The code below re-syncs backend_id properties of EdgeDB builtin
# types with the actual OIDs in the DB.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
bootstrap_mode=True,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
UPDATE schema::ScalarType
FILTER .builtin AND NOT .is_abstract
SET {
backend_id := sys::_get_pg_type_for_scalar_type(.id)
}
''',
expected_cardinality_one=False,
single_statement=True,
)
await conn.execute(sql)
if not in_dev_mode and testmode:
# Running tests on a production build.
stdlib, testmode_sql = await _amend_stdlib(
s_std.get_std_module_text('_testmode'),
stdlib,
)
await conn.execute(testmode_sql)
await metaschema.generate_support_views(
cluster,
conn,
stdlib.reflschema,
)
# Make sure that schema backend_id properties are in sync with
# the database.
compiler = edbcompiler.new_compiler(
std_schema=stdlib.stdschema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
bootstrap_mode=True,
)
_, sql = compile_bootstrap_script(
compiler,
stdlib.reflschema,
'''
SELECT schema::ScalarType {
id,
backend_id,
} FILTER .builtin AND NOT .is_abstract;
''',
expected_cardinality_one=False,
single_statement=True,
)
schema = stdlib.stdschema
typemap = await conn.fetchval(sql)
for entry in json.loads(typemap):
t = schema.get_by_id(uuidgen.UUID(entry['id']))
schema = t.set_field_value(schema, 'backend_id', entry['backend_id'])
stdlib = stdlib._replace(stdschema=schema)
if not cache_hit and (in_dev_mode or specified_cache_dir):
buildmeta.write_data_cache(
stdlib,
src_hash,
stdlib_cache,
target_dir=cache_dir,
)
await _store_static_bin_cache(
cluster,
'stdschema',
pickle.dumps(schema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'reflschema',
pickle.dumps(stdlib.reflschema, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_bin_cache(
cluster,
'classlayout',
pickle.dumps(stdlib.classlayout, protocol=pickle.HIGHEST_PROTOCOL),
)
await _store_static_json_cache(
cluster,
'introquery',
json.dumps(stdlib.introquery),
)
await metaschema.generate_support_views(cluster, conn, stdlib.reflschema)
await metaschema.generate_support_functions(conn, stdlib.reflschema)
compiler = edbcompiler.new_compiler(
std_schema=schema,
reflection_schema=stdlib.reflschema,
schema_class_layout=stdlib.classlayout,
bootstrap_mode=True,
)
return schema, stdlib.reflschema, compiler
