def generate_types():
"""
Generates and writes the C++ code for internal thing types, such as the text and number classes
Additionally, writes the symbol maps for the generated types
"""
for name, path in definitions.INTERNAL_SOURCES.items():
ast = pipeline.preprocess(SourceContext(path))
symbol_map = SymbolMapper(ast)[name]
symbol_map.convention = Symbol.INTERNAL
text_cast = CastTag(Identifier('text'))
if text_cast not in symbol_map:
symbol_map.lookup[text_cast] = Symbol.noop(text_cast,
Identifier('text'),
implicit=True)
for symbol in symbol_map:
symbol.convention = Symbol.INTERNAL
write_if_changed(
os.path.join(SYMBOLS_TARGET, f'{name}.thingsymbols'),
json.dumps(symbol_map.serialize(),
cls=JSONSerializer,
indent=4,
sort_keys=True))
def test_parameterization_propagation():
symbols = get_symbols(SOURCE_FULL)
generic_type = symbols[Identifier('Person')][Identifier(
'favorite_numbers')].type
parametrized_type = symbols[generic_type]
assert parametrized_type.name == GenericIdentifier(
Identifier('Pair'), (Identifier('number'), ))
def entry(self) -> Reference:
"""
Get the index of the program's entry point
"""
return self.resolve(
NamedAccess([Identifier('Program'),
Identifier.constructor()]), {})
def validate_thing_definition(node, name, extends=None, generics=None):
assert isinstance(node, ThingDefinition)
assert node.name == (name
if isinstance(name, Identifier) else Identifier(name))
assert node.extends == (Identifier(extends) if extends else None)
assert node.generics == ([Identifier(x)
for x in generics] if generics else None)
def test_method_parameterization():
pair_number = get_parametrized()
validate_method(pair_number[Identifier('set_values')], 'number',
['number'] * 2, 1)
validate_method(pair_number[Identifier('nested_param')],
GenericIdentifier.wrap('list', 'number'),
[GenericIdentifier.wrap('list', 'number')], 2)
def finalize_buffer(buffer: str, terminating_char, entity_class,
source_ref) -> LexicalToken:
"""
Finalize a character buffer into a lexical token
:param buffer: the characters collected thus far
:param terminating_char: the character which caused the buffer termination (not included in buffer)
:param entity_class: the current entity being collected (generally, a type of quote, or none)
:param source_ref: a reference to the source from which these tokens were derived
"""
if buffer in KEYWORDS:
return KEYWORDS[buffer](
buffer, source_ref) if KEYWORDS[buffer].EMITTABLE else None
if buffer.isdigit():
return NumericValue(buffer, source_ref)
if terminating_char == '"':
if entity_class is not LexicalQuote:
raise ValueError("Unexpected end of string")
return InlineString(buffer, source_ref)
if terminating_char == '`':
if entity_class is not LexicalBacktick:
raise ValueError("Unexpected end of inline code")
return InlineString(buffer, source_ref)
if Identifier.validate(buffer):
if entity_class in (LexicalQuote, LexicalBacktick):
raise ValueError("String was not closed")
return Identifier(buffer, source_ref)
if buffer:
raise ValueError('Lexer: cannot terminate group "{}" (at {})'.format(
buffer, source_ref))
def test_iteration_loop_parsing():
loop = parse_local('for number n in numbers')
assert isinstance(loop, IterationLoop)
assert loop.target == Identifier('n')
assert loop.target_type == Identifier('number')
assert loop.collection == Identifier('numbers')
def internal_call(target):
return OpcodeCallInternal.from_reference(
SYMBOL_MAPPER.resolve_named([
CastTag(Identifier(x[3:]))
if x.startswith('as ') else Identifier(x)
for x in target.split('.')
],
generic_validation=False))
def test_person_member_symbol_description():
symbols = get_symbols(SOURCE_PERSON)
person = symbol_map_sanity(symbols, 'Person',
('name', 'age', 'location', 'walk_to',
'say_hello', 'shout', 'favorite_numbers'))
validate_member(person[Identifier('name')], Identifier('text'), 0)
validate_member(person[Identifier('location')], Identifier('Location'), 2)
def finalize(self):
super().finalize()
if Identifier.constructor() not in self.names: # Add implicit constructor
self.children.insert(0, MethodDefinition.empty_constructor(self))
if self.extends and self.extends.untyped in definitions.INTERNAL_SOURCES:
self.children.insert(0, MemberDefinition(Identifier.super(), self.extends).deriving_from(self))
def validate_method(method: Symbol, type, arguments, index, static=False):
assert method.type == (Identifier(type) if isinstance(type, str) else type), (method.type, type)
assert method.index == index
assert method.arguments == [Identifier(x) if isinstance(x, str) else x for x in arguments]
assert method.static is static
assert method.kind is Symbol.METHOD
assert method.visibility is Symbol.PUBLIC
def test_chained_call():
call = parse_local('counter.increment().add(10)')
validate_types(call.arguments, [NumericValue])
assert call.target == NamedAccess([
MethodCall(NamedAccess([
Identifier('counter'), Identifier('increment')
])),
Identifier('add')
])
def validate_assignment(node, type, name, value):
assert isinstance(node, AssignmentOperation)
assert node.name == Identifier(name)
assert node.name.type == (Identifier(type) if type else None)
assert node.intent == (AssignmentOperation.DECELERATION
if type else AssignmentOperation.REASSIGNMENT)
if value in (MethodCall, BinaryOperation):
assert isinstance(node.value, value)
def test_generic_parsing():
node = parse_local('list<number> l = [1, 2, 3]')
assert isinstance(node, AssignmentOperation)
assert isinstance(node.name, Identifier)
assert isinstance(node.name.type, GenericIdentifier)
assert node.name == Identifier('l')
assert node.name.type.value == Identifier('list')
assert node.name.type.generics == (Identifier('number'), )
def load_identifier(value):
"""
Parse a generic identifier
"""
if isinstance(value, str):
return Identifier(value)
elif isinstance(value, list):
return GenericIdentifier(Identifier(value[0]), tuple(Identifier(x) for x in value[1]))
elif isinstance(value, dict) and value['intent'] == 'cast':
return CastTag(Symbol.load_identifier(value['type']))
def test_nested_member_parameterization():
pair_number = get_parametrized()
validate_member(pair_number[Identifier('parts')],
GenericIdentifier.wrap('list', 'number'), 2)
validate_member(
pair_number[Identifier('nested')],
GenericIdentifier.wrap(
'list',
GenericIdentifier.wrap('list',
GenericIdentifier.wrap('list', 'number'))),
3)
def test_argument_count_mismatch():
with pytest.raises(NoMatchingOverload) as e:
pipeline.compile(
SourceContext.wrap(BASE.format(code="self.no_args(1)")))
assert e.value.methods[0].name == Identifier(
'no_args') and e.value.arguments == [NumericValue(1)]
with pytest.raises(NoMatchingOverload) as e:
pipeline.compile(
SourceContext.wrap(BASE.format(code="self.two_args(1)")))
assert e.value.methods[0].name == Identifier(
'two_args') and e.value.arguments == [NumericValue(1)]
def compile(self, context: CompilationBuffer):
if not self.values:
return
buffer = context.optional()
ref = self[0].compile(buffer) # TODO: remove unnecessary recompilation of first element (used to infer type)
list_type = GenericIdentifier(Identifier('list'), (ref.type,))
last_call = MethodCall(NamedAccess([list_type, Identifier.constructor()])).deriving_from(self)
for value in self: # TODO: validate list is homogeneous, and descend to lowest common type
last_call = MethodCall(NamedAccess([last_call, Identifier("append")]), ArgumentList([value])).deriving_from(self)
return last_call.compile(context)
def get_selection(*target_types):
selector = BASE.element.selector(CONTEXT)
for target_type in target_types:
selector.constraint(Reference(Identifier(target_type)))
return selector.disambiguate(None)
def validate_member(member: Symbol, type, index, static=False):
assert member.type == (Identifier(type) if isinstance(type, str) else type)
assert member.index == index
assert member.static is static
assert member.kind is Symbol.MEMBER
assert member.visibility is Symbol.PUBLIC
def resolve(
self,
target: Union[Identifier, NamedAccess],
method_locals: dict = (),
current_generics: list = ()
) -> Reference:
"""
Resolve a reference into a Reference object
:param target: the reference being resolved (can be either an identifier or access object)
:param method_locals: the locals of the method being compiled
:return: new Reference
"""
assert not target.STATIC
if isinstance(target, Identifier):
if target in current_generics:
return Reference(Identifier.object())
elif target.untyped in self: # TODO: verify name collisions
return Reference(target)
else:
local = method_locals[target]
if not local.allowed:
raise SelfInStaticMethod(target)
if local.type not in current_generics and self[
local.
type].generics and local.source != 'self': # Check if any generic type parameters have been left unfilled
raise UnfilledGenericParameters(target, self[local.type],
None)
return LocalReference(method_locals[target], target)
elif isinstance(target, NamedAccess):
return self.resolve_named(target, method_locals, current_generics)
raise Exception("Unknown reference type {}".format(target))
def resolve_named(self,
target: Sequence,
method_locals=(),
current_generics=(),
generic_validation=True) -> ElementReference:
"""
Resolves an identifier pair (a.b) into an element reference
:param target: the Access object to resolve
:param method_locals: the current method's locals
:return: new ElementReference
"""
assert len(target) == 2
first, second, local = target[0], target[1], None
if first.STATIC:
container = self[first.type]
elif isinstance(first, IndexedAccess):
container = self.resolve_indexed(first, method_locals)
elif first.untyped in current_generics: # TODO: what about name collisions?
container = self[Identifier.object(
)] # TODO: implement `with type T implement Interface`
elif first.untyped in self.maps:
container = self[first]
elif first in method_locals:
local = method_locals[first]
if not local.allowed:
raise SelfInStaticMethod(target)
container = self[local.type]
else:
raise Exception(
'Cannot resolve first level access {} (on {}) from {}'.format(
first, first.source_ref, method_locals))
container, element = self.pull(container, second, target)
remaining_generics = set(container.generics) - set(
current_generics
) # TODO: are we sure generic name conflicts don't prevent this validation?
if generic_validation and not element.is_complete({
x: Identifier.invalid()
for x in remaining_generics
}): # Check if any generic type parameters have been left unfilled
raise UnfilledGenericParameters(target, container, element)
return ElementReference(container, self.index(container), element,
local)
def from_serialized(cls, code, argument_names, argument_types):
ast = preprocess.preprocess(SourceContext.wrap(code))
return cls(
ast.children,
ArgumentList([
Identifier(name, type_name=arg_type)
for name, arg_type in zip(argument_names, argument_types)
]))
def compile(
self, context: CompilationBuffer
): # TODO: we should probably reparse the maps into identifiers
method_call = MethodCall(
NamedAccess.extend(self.lhs,
Identifier(self.operator.serialize())),
[self.rhs])
return method_call.compile(context)
def __init__(self, target: Identifier, target_type: Identifier,
collection: ValueType):
super().__init__(None, (target, target_type, collection))
self.target, self.target_type, self.collection = target, target_type, collection
self.iterator_id = next(IterationLoop.TRANSIENT_COUNTER)
self.iterator = self.iterator_container_name[0]
self.continuation_check = MethodCall(
NamedAccess.extend(self.iterator,
Identifier('has_next'))).deriving_from(self)
self.continuation_next = MethodCall(
NamedAccess.extend(self.iterator,
Identifier('next'))).deriving_from(self)
self.value = self.continuation_check
if isinstance(self.collection, MethodCall):
self.collection.is_captured = True
def __init__(self, method: MethodDefinition, thing: ThingDefinition):
super(IndexerContext, self).__init__()
self.current_method = method
self.locals = OrderedDict({Identifier.self(): LocalMember(thing.name, 0, 'self', not method.static)})
# Note: this keeps the first slot reserved for self, even in a static method
for arg in method.arguments:
self.locals[arg] = LocalMember(arg.type, len(self.locals), 'argument', True)
def normalize_id(param):
if isinstance(param, str):
return Identifier(param)
if isinstance(param, int):
return NumericValue(param)
if isinstance(param, (tuple, list)):
return [normalize_id(x) for x in param]
return param
def validate_method_definition(node,
name,
expected_arguments=(),
return_type=None):
assert isinstance(node, MethodDefinition)
assert node.name == (name
if isinstance(name, Identifier) else Identifier(name))
assert node.return_type == return_type
for actual_argument, expected_argument in zip(node.arguments,
expected_arguments):
assert actual_argument.value == expected_argument[0]
assert actual_argument.type.value == expected_argument[1]
def finalize(self):
if not self.is_constructor():
return super().finalize()
for descendant in self.descendants:
if isinstance(
descendant,
MethodCall) and descendant.target[0] == Identifier.super():
descendant.replace(
AssignmentOperation(
AssignmentOperation.REASSIGNMENT,
NamedAccess([Identifier.self(),
Identifier.super()]),
MethodCall(NamedAccess(
[self.parent.extends,
Identifier.constructor()]),
descendant.arguments,
is_captured=True).deriving_from(
self)).deriving_from(descendant))
super().finalize()
def compile(self, context: CompilationBuffer):
iterator_name, iterator_type = self.iterator_container_name
AssignmentOperation(
AssignmentOperation.REASSIGNMENT,
iterator_name,
MethodCall(NamedAccess.extend(self.collection,
Identifier('iterator')),
is_captured=True).deriving_from(self),
iterator_type,
).deriving_from(self).compile(context)
super().compile(context)