def delta_from_ddl(stmts,
*,
schema,
modaliases,
stdmode: bool = False,
testmode: bool = False):
alter_db = s_delta.DeltaRoot()
context = s_delta.CommandContext()
context.modaliases = modaliases
context.schema = schema
context.stdmode = stdmode
context.testmode = testmode
if isinstance(stmts, edgeql.ast.Base):
stmts = [stmts]
for stmt in stmts:
with context(s_delta.DeltaRootContext(alter_db)):
alter_db.add(
cmd_from_ddl(stmt,
context=context,
schema=schema,
modaliases=modaliases,
testmode=testmode))
return alter_db
def run_ddl(cls, schema, ddl, default_module=defines.DEFAULT_MODULE_ALIAS):
statements = edgeql.parse_block(ddl)
current_schema = schema
target_schema = None
for stmt in statements:
if isinstance(stmt, qlast.CreateDelta):
# CREATE MIGRATION
if target_schema is None:
target_schema = _load_std_schema()
ddl_plan = s_ddl.cmd_from_ddl(
stmt,
schema=current_schema,
modaliases={None: default_module},
testmode=True)
ddl_plan = s_ddl.compile_migration(ddl_plan, target_schema,
current_schema)
elif isinstance(stmt, qlast.Delta):
# APPLY MIGRATION
delta_cmd = s_ddl.cmd_from_ddl(
stmt,
schema=current_schema,
modaliases={None: default_module},
testmode=True)
delta = current_schema.get(delta_cmd.classname)
ddl_plan = sd.DeltaRoot(canonical=True)
ddl_plan.update(delta.get_commands(current_schema))
elif isinstance(stmt, qlast.DDL):
# CREATE/DELETE/ALTER (FUNCTION, TYPE, etc)
ddl_plan = s_ddl.delta_from_ddl(
stmt,
schema=current_schema,
modaliases={None: default_module},
testmode=True)
else:
raise ValueError(
f'unexpected {stmt!r} in compiler setup script')
context = sd.CommandContext()
context.testmode = True
current_schema, _ = ddl_plan.apply(current_schema, context)
return current_schema
def _assert_migration_consistency(self, schema_text):
migration_text = f'''
CREATE MIGRATION m TO {{
{schema_text}
}};
'''
migration_ql = edgeql.parse_block(migration_text)
migration_cmd = s_ddl.cmd_from_ddl(
migration_ql[0],
schema=self.schema,
modaliases={None: 'default'},
)
migration_cmd = s_ddl.compile_migration(
migration_cmd,
self.std_schema,
self.schema,
)
context = s_delta.CommandContext()
schema, migration = migration_cmd.apply(self.schema, context)
ddl_plan = s_delta.DeltaRoot(canonical=True)
ddl_plan.update(migration.get_commands(schema))
baseline_schema, _ = ddl_plan.apply(schema, context)
ddl_text = s_ddl.ddl_text_from_delta(schema, migration)
try:
test_schema = self.run_ddl(schema, ddl_text)
except errors.EdgeDBError as e:
self.fail(markup.dumps(e))
diff = s_ddl.delta_schemas(baseline_schema, test_schema)
if list(diff.get_subcommands()):
self.fail(
f'unexpected difference in schema produced by\n'
f'COMMIT MIGRATION and DDL obtained from GET MIGRATION:\n'
f'{markup.dumps(diff)}\n'
f'DDL text was:\n{ddl_text}')
def _compile_and_apply_delta_command(self, ctx: CompileContext,
cmd) -> dbstate.BaseQuery:
current_tx = ctx.state.current_tx()
schema = current_tx.get_schema()
context = self._new_delta_context(ctx)
if current_tx.is_implicit():
if isinstance(cmd, s_deltas.CreateDelta):
command = 'CREATE MIGRATION'
elif isinstance(cmd, s_deltas.GetDelta):
command = 'GET MIGRATION'
else:
command = 'COMMIT MIGRATION'
raise errors.QueryError(
f'{command} must be executed in a transaction block')
if isinstance(cmd, s_deltas.CreateDelta):
delta = None
else:
delta = schema.get(cmd.classname)
with context(s_deltas.DeltaCommandContext(schema, cmd, delta)):
if isinstance(cmd, s_deltas.CommitDelta):
ddl_plan = s_delta.DeltaRoot(canonical=True)
ddl_plan.update(delta.get_commands(schema))
return self._compile_and_apply_ddl_command(ctx, ddl_plan)
elif isinstance(cmd, s_deltas.GetDelta):
delta_ql = s_ddl.ddl_text_from_delta(schema, delta)
query_ql = qlast.SelectQuery(result=qlast.StringConstant(
quote="'", value=ql_quote.escape_string(delta_ql)))
return self._compile_ql_query(ctx, query_ql)
elif isinstance(cmd, s_deltas.CreateDelta):
schema, _ = cmd.apply(schema, context)
current_tx.update_schema(schema)
# We must return *some* SQL; return a no-op command.
return dbstate.DDLQuery(sql=(b'SELECT NULL LIMIT 0;', ))
else:
raise errors.InternalServerError(
f'unexpected delta command: {cmd!r}') # pragma: no cover
def generate_structure(schema: s_schema.Schema) -> SchemaReflectionParts:
"""Generate schema reflection structure from Python schema classes.
Returns:
A quadruple (as a SchemaReflectionParts instance) containing:
- Delta, which, when applied to stdlib, yields an enhanced
version of the `schema` module that contains all types
and properties, not just those that are publicly exposed
for introspection.
- A mapping, containing type layout description for all
schema classes.
- A sequence of EdgeQL queries necessary to introspect
a database schema.
- A sequence of EdgeQL queries necessary to introspect
global objects, such as roles and databases.
"""
delta = sd.DeltaRoot()
classlayout: Dict[Type[s_obj.Object], SchemaTypeLayout, ] = {}
ordered_link = schema.get('schema::ordered', type=s_links.Link)
py_classes = []
schema = _run_ddl(
'''
CREATE FUNCTION sys::_get_pg_type_for_scalar_type(
typeid: std::uuid
) -> std::int64 {
USING SQL $$
SELECT
coalesce(
(
SELECT
tn::regtype::oid
FROM
edgedb._get_base_scalar_type_map()
AS m(tid uuid, tn text)
WHERE
m.tid = "typeid"
),
(
SELECT
typ.oid
FROM
pg_catalog.pg_type typ
WHERE
typ.typname = "typeid"::text || '_domain'
),
edgedb.raise(
NULL::bigint,
'invalid_parameter_value',
msg => (
'cannot determine OID of '
|| typeid::text
)
)
)::bigint
$$;
SET volatility := 'STABLE';
};
CREATE FUNCTION sys::_expr_from_json(
data: json
) -> OPTIONAL tuple<text: str, refs: array<uuid>> {
USING SQL $$
SELECT
"data"->>'text' AS text,
coalesce(r.refs, ARRAY[]::uuid[]) AS refs
FROM
(SELECT
array_agg(v::uuid) AS refs
FROM
jsonb_array_elements_text("data"->'refs') AS v
) AS r
WHERE
jsonb_typeof("data") != 'null'
$$;
SET volatility := 'IMMUTABLE';
};
''',
schema=schema,
delta=delta,
)
for py_cls in s_obj.ObjectMeta.get_schema_metaclasses():
if isinstance(py_cls, adapter.Adapter):
continue
if py_cls is s_obj.GlobalObject:
continue
py_classes.append(py_cls)
read_sets: Dict[Type[s_obj.Object], List[str]] = {}
for py_cls in py_classes:
rschema_name = get_schema_name_for_pycls(py_cls)
schema_objtype = schema.get(
rschema_name,
type=s_objtypes.ObjectType,
default=None,
)
bases = []
for base in py_cls.__bases__:
if base in py_classes:
bases.append(get_schema_name_for_pycls(base))
default_base = get_default_base_for_pycls(py_cls)
if not bases and rschema_name != default_base:
bases.append(default_base)
reflection = py_cls.get_reflection_method()
is_simple_wrapper = issubclass(py_cls, s_types.CollectionExprAlias)
if schema_objtype is None:
as_abstract = (reflection is s_obj.ReflectionMethod.REGULAR
and not is_simple_wrapper)
schema = _run_ddl(
f'''
CREATE {'ABSTRACT' if as_abstract else ''}
TYPE {rschema_name}
EXTENDING {', '.join(str(b) for b in bases)};
''',
schema=schema,
delta=delta,
)
schema_objtype = schema.get(rschema_name,
type=s_objtypes.ObjectType)
else:
ex_bases = schema_objtype.get_bases(schema).names(schema)
_, added_bases = s_inh.delta_bases(ex_bases, bases)
if added_bases:
for subset, position in added_bases:
if isinstance(position, tuple):
position_clause = (f'{position[0]} {position[1].name}')
else:
position_clause = position
bases_expr = ', '.join(str(t.name) for t in subset)
stmt = f'''
ALTER TYPE {rschema_name} {{
EXTENDING {bases_expr} {position_clause}
}}
'''
schema = _run_ddl(
stmt,
schema=schema,
delta=delta,
)
if reflection is s_obj.ReflectionMethod.NONE:
continue
referrers = py_cls.get_referring_classes()
if reflection is s_obj.ReflectionMethod.AS_LINK:
if not referrers:
raise RuntimeError(
f'schema class {py_cls.__name__} is declared with AS_LINK '
f'reflection method but is not referenced in any RefDict')
is_concrete = not schema_objtype.get_abstract(schema)
if (is_concrete and not is_simple_wrapper
and any(not b.get_abstract(schema) for b in
schema_objtype.get_ancestors(schema).objects(schema))):
raise RuntimeError(
f'non-abstract {schema_objtype.get_verbosename(schema)} has '
f'non-abstract ancestors')
read_shape = read_sets[py_cls] = []
if is_concrete:
read_shape.append(
'_tname := .__type__[IS schema::ObjectType].name')
classlayout[py_cls] = {}
ownfields = py_cls.get_ownfields()
for fn, field in py_cls.get_fields().items():
if (field.ephemeral or (field.reflection_method
is not s_obj.ReflectionMethod.REGULAR)):
continue
storage = _classify_object_field(field)
ptr = schema_objtype.maybe_get_ptr(schema, sn.UnqualName(fn))
if fn in ownfields:
qual = "REQUIRED" if field.required else "OPTIONAL"
if ptr is None:
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
CREATE {qual}
{storage.ptrkind} {fn} -> {storage.ptrtype};
}}
''',
schema=schema,
delta=delta,
)
ptr = schema_objtype.getptr(schema, sn.UnqualName(fn))
if storage.shadow_ptrkind is not None:
pn = f'{fn}__internal'
internal_ptr = schema_objtype.maybe_get_ptr(
schema, sn.UnqualName(pn))
if internal_ptr is None:
ptrkind = storage.shadow_ptrkind
ptrtype = storage.shadow_ptrtype
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
CREATE {qual}
{ptrkind} {pn} -> {ptrtype};
}}
''',
schema=schema,
delta=delta,
)
else:
assert ptr is not None
if is_concrete:
read_ptr = fn
if field.type_is_generic_self:
read_ptr = f'{read_ptr}[IS {rschema_name}]'
if field.reflection_proxy:
proxy_type, proxy_link = field.reflection_proxy
read_ptr = (
f'{read_ptr}: {{name, value := .{proxy_link}.id}}')
if ptr.issubclass(schema, ordered_link):
read_ptr = f'{read_ptr} ORDER BY @index'
read_shape.append(read_ptr)
if storage.shadow_ptrkind is not None:
read_shape.append(f'{fn}__internal')
if field.reflection_proxy:
proxy_type_name, proxy_link_name = field.reflection_proxy
proxy_obj = schema.get(proxy_type_name,
type=s_objtypes.ObjectType)
proxy_link_obj = proxy_obj.getptr(
schema, sn.UnqualName(proxy_link_name))
tgt = proxy_link_obj.get_target(schema)
else:
tgt = ptr.get_target(schema)
assert tgt is not None
cardinality = ptr.get_cardinality(schema)
assert cardinality is not None
classlayout[py_cls][fn] = SchemaFieldDesc(
fieldname=fn,
type=tgt,
cardinality=cardinality,
properties={},
storage=storage,
is_ordered=ptr.issubclass(schema, ordered_link),
reflection_proxy=field.reflection_proxy,
)
# Second pass: deal with RefDicts, which are reflected as links.
for py_cls in py_classes:
rschema_name = get_schema_name_for_pycls(py_cls)
schema_cls = schema.get(rschema_name, type=s_objtypes.ObjectType)
for refdict in py_cls.get_own_refdicts().values():
ref_ptr = schema_cls.maybe_get_ptr(schema,
sn.UnqualName(refdict.attr))
ref_cls = refdict.ref_cls
assert issubclass(ref_cls, s_obj.Object)
shadow_ref_ptr = None
reflect_as_link = (ref_cls.get_reflection_method() is
s_obj.ReflectionMethod.AS_LINK)
if reflect_as_link:
reflection_link = ref_cls.get_reflection_link()
assert reflection_link is not None
target_field = ref_cls.get_field(reflection_link)
target_cls = target_field.type
shadow_pn = f'{refdict.attr}__internal'
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
CREATE OPTIONAL MULTI LINK {shadow_pn}
EXTENDING schema::reference
-> {get_schema_name_for_pycls(ref_cls)} {{
ON TARGET DELETE ALLOW;
}};
}}
''',
schema=schema,
delta=delta,
)
shadow_ref_ptr = schema_cls.getptr(schema,
sn.UnqualName(shadow_pn))
else:
target_cls = ref_cls
if ref_ptr is None:
ptr_type = get_schema_name_for_pycls(target_cls)
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
CREATE OPTIONAL MULTI LINK {refdict.attr}
EXTENDING schema::reference
-> {ptr_type} {{
ON TARGET DELETE ALLOW;
}};
}}
''',
schema=schema,
delta=delta,
)
ref_ptr = schema_cls.getptr(schema,
sn.UnqualName(refdict.attr))
else:
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
ALTER LINK {refdict.attr}
ON TARGET DELETE ALLOW;
}}
''',
schema=schema,
delta=delta,
)
assert isinstance(ref_ptr, s_links.Link)
if py_cls not in classlayout:
classlayout[py_cls] = {}
# First, fields declared to be reflected as link properties.
props = _get_reflected_link_props(
ref_ptr=ref_ptr,
target_cls=ref_cls,
schema=schema,
)
if reflect_as_link:
# Then, because it's a passthrough reflection, all scalar
# fields of the proxy object.
fields_as_props = [
f for f in ref_cls.get_ownfields().values()
if (not f.ephemeral and (
f.reflection_method is not s_obj.ReflectionMethod.
AS_LINK) and f.name != refdict.backref_attr
and f.name != ref_cls.get_reflection_link())
]
extra_props = _classify_scalar_object_fields(fields_as_props)
for fn, storage in {**props, **extra_props}.items():
prop_ptr = ref_ptr.maybe_get_ptr(schema, sn.UnqualName(fn))
if prop_ptr is None:
pty = storage.ptrtype
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
ALTER LINK {refdict.attr} {{
CREATE OPTIONAL PROPERTY {fn} -> {pty};
}}
}}
''',
schema=schema,
delta=delta,
)
if shadow_ref_ptr is not None:
assert isinstance(shadow_ref_ptr, s_links.Link)
shadow_pn = shadow_ref_ptr.get_shortname(schema).name
for fn, storage in props.items():
prop_ptr = shadow_ref_ptr.maybe_get_ptr(
schema, sn.UnqualName(fn))
if prop_ptr is None:
pty = storage.ptrtype
schema = _run_ddl(
f'''
ALTER TYPE {rschema_name} {{
ALTER LINK {shadow_pn} {{
CREATE OPTIONAL PROPERTY {fn} -> {pty};
}}
}}
''',
schema=schema,
delta=delta,
)
for py_cls in py_classes:
rschema_name = get_schema_name_for_pycls(py_cls)
schema_cls = schema.get(rschema_name, type=s_objtypes.ObjectType)
is_concrete = not schema_cls.get_abstract(schema)
read_shape = read_sets[py_cls]
for refdict in py_cls.get_refdicts():
if py_cls not in classlayout:
classlayout[py_cls] = {}
ref_ptr = schema_cls.getptr(schema,
sn.UnqualName(refdict.attr),
type=s_links.Link)
tgt = ref_ptr.get_target(schema)
assert tgt is not None
cardinality = ref_ptr.get_cardinality(schema)
assert cardinality is not None
classlayout[py_cls][refdict.attr] = SchemaFieldDesc(
fieldname=refdict.attr,
type=tgt,
cardinality=cardinality,
properties={},
is_ordered=ref_ptr.issubclass(schema, ordered_link),
reflection_proxy=None,
is_refdict=True,
)
target_cls = refdict.ref_cls
props = _get_reflected_link_props(
ref_ptr=ref_ptr,
target_cls=target_cls,
schema=schema,
)
reflect_as_link = (target_cls.get_reflection_method() is
s_obj.ReflectionMethod.AS_LINK)
prop_layout = {}
extra_prop_layout = {}
for fn, storage in props.items():
prop_ptr = ref_ptr.getptr(schema, sn.UnqualName(fn))
prop_tgt = prop_ptr.get_target(schema)
assert prop_tgt is not None
prop_layout[fn] = (prop_tgt, storage.fieldtype)
if reflect_as_link:
# Then, because it's a passthrough reflection, all scalar
# fields of the proxy object.
fields_as_props = [
f for f in target_cls.get_ownfields().values()
if (not f.ephemeral and (
f.reflection_method is not s_obj.ReflectionMethod.
AS_LINK) and f.name != refdict.backref_attr
and f.name != target_cls.get_reflection_link())
]
extra_props = _classify_scalar_object_fields(fields_as_props)
for fn, storage in extra_props.items():
prop_ptr = ref_ptr.getptr(schema, sn.UnqualName(fn))
prop_tgt = prop_ptr.get_target(schema)
assert prop_tgt is not None
extra_prop_layout[fn] = (prop_tgt, storage.fieldtype)
else:
extra_prop_layout = {}
classlayout[py_cls][refdict.attr].properties.update({
**prop_layout,
**extra_prop_layout,
})
if reflect_as_link:
shadow_tgt = schema.get(
get_schema_name_for_pycls(ref_cls),
type=s_objtypes.ObjectType,
)
classlayout[py_cls][f'{refdict.attr}__internal'] = (
SchemaFieldDesc(
fieldname=refdict.attr,
type=shadow_tgt,
cardinality=qltypes.SchemaCardinality.Many,
properties=prop_layout,
is_refdict=True,
))
if is_concrete:
read_ptr = refdict.attr
prop_shape_els = []
if reflect_as_link:
read_ptr = f'{read_ptr}__internal'
ref_ptr = schema_cls.getptr(
schema,
sn.UnqualName(f'{refdict.attr}__internal'),
)
for fn in props:
prop_shape_els.append(f'@{fn}')
if prop_shape_els:
prop_shape = ',\n'.join(prop_shape_els)
read_ptr = f'{read_ptr}: {{id, {prop_shape}}}'
if ref_ptr.issubclass(schema, ordered_link):
read_ptr = f'{read_ptr} ORDER BY @index'
read_shape.append(read_ptr)
local_parts = []
global_parts = []
for py_cls, shape_els in read_sets.items():
if (not shape_els
# The CollectionExprAlias family needs to be excluded
# because TupleExprAlias and ArrayExprAlias inherit from
# concrete classes and so are picked up from those.
or issubclass(py_cls, s_types.CollectionExprAlias)):
continue
rschema_name = get_schema_name_for_pycls(py_cls)
shape = ',\n'.join(shape_els)
qry = f'''
SELECT {rschema_name} {{
{shape}
}}
'''
if not issubclass(py_cls, (s_types.Collection, s_obj.GlobalObject)):
qry += ' FILTER NOT .builtin'
if issubclass(py_cls, s_obj.GlobalObject):
global_parts.append(qry)
else:
local_parts.append(qry)
delta.canonical = True
return SchemaReflectionParts(
intro_schema_delta=delta,
class_layout=classlayout,
local_intro_parts=local_parts,
global_intro_parts=global_parts,
)
