def assign(target, value, context, warnings, generator=False):
value_type = expr.visit_expression(value, Unknown(), context, warnings)
static_value = static_evaluate(value, context)
if generator:
if isinstance(value_type, (List, Set)):
assign_type = value_type.item_type
elif isinstance(value_type, Tuple):
assign_type = Unknown() # TODO
else:
assign_type = Unknown()
else:
assign_type = value_type
target_token = expr.get_token(target)
if target_token in ('Tuple', 'List'):
values = static_value if isinstance(static_value,
(Tuple, List)) else []
assign_values = chain(values, repeat(UnknownValue()))
if isinstance(assign_type, Tuple):
assign_types = chain(assign_type.item_types, repeat(Unknown()))
elif isinstance(assign_type, (List, Set)):
assign_types = repeat(assign_type.item_type)
else:
assign_types = repeat(Unknown())
return [
assign_single_target(target, assign_type, static_value, context)
for target, assign_type, static_value
in zip(target.elts, assign_types, assign_values)]
else:
return [assign_single_target(target, assign_type,
static_value, context)]
def assign(target, value, context, warnings, generator=False):
value_type = expr.visit_expression(value, Unknown(), context, warnings)
static_value = static_evaluate(value, context)
if generator:
if isinstance(value_type, (List, Set)):
assign_type = value_type.item_type
elif isinstance(value_type, Tuple):
assign_type = Unknown() # TODO
else:
assign_type = Unknown()
else:
assign_type = value_type
target_token = expr.get_token(target)
if target_token in ('Tuple', 'List'):
values = static_value if isinstance(static_value,
(Tuple, List)) else []
assign_values = chain(values, repeat(UnknownValue()))
if isinstance(assign_type, Tuple):
assign_types = chain(assign_type.item_types, repeat(Unknown()))
elif isinstance(assign_type, (List, Set)):
assign_types = repeat(assign_type.item_type)
else:
assign_types = repeat(Unknown())
return [
assign_single_target(target, assign_type, static_value, context)
for target, assign_type, static_value in zip(
target.elts, assign_types, assign_values)
]
else:
return [
assign_single_target(target, assign_type, static_value, context)
]
def maybe_inferences(test, context):
types = {name: context.get_type(name) for name in get_names(test)}
maybes = {k: v for k, v in types.items() if isinstance(v, Maybe)}
if_inferences = {}
else_inferences = {}
for name, maybe_type in maybes.items():
context.begin_scope()
context.add(Symbol(name, NoneType(), None))
none_value = static_evaluate(test, context)
context.end_scope()
if none_value is False:
if_inferences[name] = maybe_type.subtype
if none_value is True:
else_inferences[name] = maybe_type.subtype
context.begin_scope()
context.add(Symbol(name, maybe_type.subtype, UnknownValue()))
non_none_value = static_evaluate(test, context)
context.end_scope()
if non_none_value is False:
if_inferences[name] = NoneType()
if non_none_value is True:
else_inferences[name] = NoneType()
return if_inferences, else_inferences
def _visit_expression(node, expected_type, context, warnings):
recur = partial(visit_expression, context=context, warnings=warnings)
probe = partial(expression_type, context=context)
comp = partial(comprehension_type, context=context, warnings=warnings)
token = get_token(node)
if token == 'BoolOp':
for expr in node.values:
recur(expr, Bool())
return Bool() # more restrictive than Python
if token == 'BinOp':
operator = get_token(node.op)
if operator == 'Add':
left_probe = probe(node.left)
right_probe = probe(node.right)
if isinstance(left_probe, Tuple) or isinstance(right_probe, Tuple):
left = recur(node.left, BaseTuple())
right = recur(node.right, BaseTuple())
if isinstance(left, Tuple) and isinstance(right, Tuple):
return Tuple(left.item_types + right.item_types)
else:
return Unknown()
union_type = Union(Num(), Str(), List(Unknown()))
left_intersect = type_intersection(left_probe, union_type)
right_intersect = type_intersection(right_probe, union_type)
sub_intersect = type_intersection(left_intersect, right_intersect)
full_intersect = type_intersection(expected_type, sub_intersect)
intersect = (full_intersect or sub_intersect or left_intersect
or right_intersect or union_type)
recur(node.left, intersect)
recur(node.right, intersect)
return intersect
elif operator == 'Mult':
union_type = Union(Num(), Str())
expected_intersect = type_intersection(expected_type, union_type)
left_intersect = type_intersection(probe(node.left), union_type)
right = recur(node.right, Num())
if isinstance(left_intersect, Num):
recur(node.left, Num())
return Num()
elif isinstance(left_intersect, Str):
recur(node.left, Str())
return Str()
elif isinstance(expected_intersect, Num):
recur(node.left, Num())
return Num()
elif isinstance(expected_intersect, Str):
recur(node.left, Str())
return Str()
else:
recur(node.left, union_type)
return union_type
elif operator == 'Mod':
# num % num OR str % unknown
union_type = Union(Num(), Str())
expected_intersect = type_intersection(expected_type, union_type)
left_intersect = type_intersection(probe(node.left), union_type)
if isinstance(left_intersect, Num):
recur(node.left, Num())
recur(node.right, Num())
return Num()
elif isinstance(left_intersect, Str):
recur(node.left, Str())
recur(node.right, Unknown())
return Str()
elif isinstance(expected_intersect, Num):
recur(node.left, Num())
recur(node.right, Num())
return Num()
elif isinstance(expected_intersect, Str):
recur(node.left, Str())
recur(node.right, Unknown())
return Str()
else:
recur(node.left, union_type)
recur(node.right, Unknown())
return union_type
else:
recur(node.left, Num())
recur(node.right, Num())
return Num()
if token == 'UnaryOp':
if get_token(node.op) == 'Not':
recur(node.operand, Bool())
return Bool()
else:
recur(node.operand, Num())
return Num()
if token == 'Lambda':
return construct_function_type(node, LambdaVisitor(context))
if token == 'IfExp':
recur(node.test, Bool())
if_inferences, else_inferences = maybe_inferences(node.test, context)
context.begin_scope(Scope(if_inferences))
body_type = recur(node.body, expected_type)
context.end_scope()
context.begin_scope(Scope(else_inferences))
else_type = recur(node.orelse, expected_type)
context.end_scope()
return unify_types([body_type, else_type])
if token == 'Dict':
key_type = unify_types([recur(key, Unknown()) for key in node.keys])
value_type = unify_types(
[recur(value, Unknown()) for value in node.values])
return Dict(key_type, value_type)
if token == 'Set':
subtype = (expected_type.item_type
if isinstance(expected_type, Set) else Unknown())
return Set(unify_types([recur(elt, Unknown()) for elt in node.elts]))
if token == 'ListComp':
subtype = (expected_type.item_type
if isinstance(expected_type, List) else Unknown())
return List(comp(node.elt, node.generators, subtype))
if token == 'SetComp':
subtype = (expected_type.item_type
if isinstance(expected_type, Set) else Unknown())
return Set(comp(node.elt, node.generators, subtype))
if token == 'DictComp':
expected_key_type = (expected_type.key_type if isinstance(
expected_type, Dict) else Unknown())
expected_value_type = (expected_type.value_type if isinstance(
expected_type, Dict) else Unknown())
key_type = comp(node.key, node.generators, expected_key_type)
value_type = comp(node.value, node.generators, expected_value_type)
return Dict(key_type, value_type)
if token == 'GeneratorExp':
subtype = (expected_type.item_type
if isinstance(expected_type, List) else Unknown())
return List(comp(node.elt, node.generators, subtype))
if token == 'Yield':
return List(recur(node.value, Unknown()))
if token == 'Compare':
operator = get_token(node.ops[0])
if len(node.ops) > 1 or len(node.comparators) > 1:
warnings.warn(node, 'comparison-operator-chaining')
if operator in ['Eq', 'NotEq', 'Lt', 'LtE', 'Gt', 'GtE']:
# all operands are constrained to have the same type
# as their intersection
left_probe = probe(node.left)
right_probe = probe(node.comparators[0])
intersection = type_intersection(left_probe, right_probe)
if intersection is None:
recur(node.left, right_probe)
recur(node.comparators[0], left_probe)
else:
recur(node.left, intersection)
recur(node.comparators[0], intersection)
if operator in ['Is', 'IsNot']:
recur(node.left, Maybe(Unknown()))
recur(node.comparators[0], NoneType())
if operator in ['In', 'NotIn']:
# constrain right to list/set of left, and left to inst. of right
left_probe = probe(node.left)
right_probe = probe(node.comparators[0])
union_type = Union(List(left_probe), Set(left_probe),
Dict(left_probe, Unknown()), Str())
recur(node.comparators[0], union_type)
if isinstance(right_probe, (List, Set)):
recur(node.left, right_probe.item_type)
elif isinstance(right_probe, Dict):
recur(node.left, right_probe.key_type)
else:
recur(node.left, Unknown())
return Bool()
if token == 'Call':
function_type = recur(node.func, Unknown())
if not isinstance(function_type, (Class, Function)):
if not isinstance(function_type, Unknown):
warnings.warn(node, 'not-a-function')
return Unknown()
signature = function_type.signature
instance = (function_type.instance if isinstance(
function_type, Function) else None)
offset = 1 if (instance is not None
or isinstance(function_type, Class)) else 0
argument_scope = Scope()
if instance is not None:
self_symbol = Symbol(signature.names[0], instance)
argument_scope.add(self_symbol)
# make sure all required arguments are specified
if node.starargs is None and node.kwargs is None:
start = offset + len(node.args)
required = signature.names[start:signature.min_count]
kwarg_names = [keyword.arg for keyword in node.keywords]
missing = [name for name in required if name not in kwarg_names]
for missing_argument in missing:
warnings.warn(node, 'missing-argument', missing_argument)
# check for too many arguments
if signature.vararg_name is None:
if len(node.args) + len(node.keywords) > len(signature.types):
warnings.warn(node, 'too-many-arguments')
# load positional arguments
for i, arg in enumerate(node.args):
if i + offset >= len(signature):
break
arg_type = recur(arg, signature.types[i + offset])
value = static_evaluate(arg, context)
argument_scope.add(
Symbol(signature.names[i + offset], arg_type, value))
# load keyword arguments
for kwarg in node.keywords:
# TODO: make sure there is no overlap with positional args
expected_type = signature.get_dict().get(kwarg.arg)
if expected_type is None:
warnings.warn(node, 'extra-keyword', kwarg.arg)
else:
arg_type = recur(kwarg.value, expected_type)
value = static_evaluate(kwarg.value, context)
argument_scope.add(Symbol(kwarg.arg, arg_type, value))
if node.starargs is not None:
recur(node.starargs, List(Unknown()))
if node.kwargs is not None:
recur(node.kwargs, Dict(Unknown(), Unknown()))
return_type, _ = function_type.evaluator.evaluate(argument_scope)
return return_type
if token == 'Repr':
return Str()
if token == 'Num':
return Num()
if token == 'Str':
return Str()
if token == 'Attribute':
value_type = recur(node.value, Unknown())
if isinstance(value_type, Unknown):
return Unknown()
if not isinstance(value_type, Instance):
warnings.warn(node, 'not-an-instance')
return Unknown()
attr_type = value_type.attributes.get_type(node.attr)
if attr_type is None:
warnings.warn(node, 'not-a-member')
return Unknown()
return attr_type
if token == 'Subscript':
union_type = Union(List(Unknown()), Dict(Unknown(), Unknown()),
BaseTuple())
value_type = recur(node.value, union_type)
if get_token(node.slice) == 'Index':
if isinstance(value_type, Tuple):
index = static_evaluate(node.slice.value, context)
if isinstance(index, UnknownValue):
return Unknown()
if not isinstance(index, int):
return Unknown()
if not 0 <= index < len(value_type.item_types):
return Unknown()
return value_type.item_types[index]
elif isinstance(value_type, List):
return value_type.item_type
elif isinstance(value_type, Dict):
return value_type.value_type
else:
return Unknown()
elif get_token(node.slice) == 'Slice':
if node.slice.lower is not None:
recur(node.slice.lower, Num())
if node.slice.upper is not None:
recur(node.slice.upper, Num())
if node.slice.step is not None:
recur(node.slice.step, Num())
return value_type
else:
return value_type
if token == 'Name':
defined_type = context.get_type(node.id)
if defined_type is None:
warnings.warn(node, 'undefined', node.id)
context.add_constraint(node.id, expected_type)
return defined_type or Unknown()
if token == 'List':
subtype = (expected_type.item_type
if isinstance(expected_type, List) else Unknown())
return List(unify_types([recur(elt, subtype) for elt in node.elts]))
if token == 'Tuple':
if (isinstance(expected_type, Tuple)
and len(node.elts) == len(expected_type.item_types)):
return Tuple([
recur(element, type_)
for element, type_ in zip(node.elts, expected_type.item_types)
])
return Tuple([recur(element, Unknown()) for element in node.elts])
raise Exception('visit_expression does not recognize ' + token)
def _visit_expression(node, expected_type, context, warnings):
recur = partial(visit_expression, context=context, warnings=warnings)
probe = partial(expression_type, context=context)
comp = partial(comprehension_type, context=context, warnings=warnings)
token = get_token(node)
if token == 'BoolOp':
for expr in node.values:
recur(expr, Bool())
return Bool() # more restrictive than Python
if token == 'BinOp':
operator = get_token(node.op)
if operator == 'Add':
left_probe = probe(node.left)
right_probe = probe(node.right)
if isinstance(left_probe, Tuple) or isinstance(right_probe, Tuple):
left = recur(node.left, BaseTuple())
right = recur(node.right, BaseTuple())
if isinstance(left, Tuple) and isinstance(right, Tuple):
return Tuple(left.item_types + right.item_types)
else:
return Unknown()
union_type = Union(Num(), Str(), List(Unknown()))
left_intersect = type_intersection(left_probe, union_type)
right_intersect = type_intersection(right_probe, union_type)
sub_intersect = type_intersection(left_intersect, right_intersect)
full_intersect = type_intersection(expected_type, sub_intersect)
intersect = (full_intersect or sub_intersect or left_intersect
or right_intersect or union_type)
recur(node.left, intersect)
recur(node.right, intersect)
return intersect
elif operator == 'Mult':
union_type = Union(Num(), Str())
expected_intersect = type_intersection(expected_type, union_type)
left_intersect = type_intersection(probe(node.left), union_type)
right = recur(node.right, Num())
if isinstance(left_intersect, Num):
recur(node.left, Num())
return Num()
elif isinstance(left_intersect, Str):
recur(node.left, Str())
return Str()
elif isinstance(expected_intersect, Num):
recur(node.left, Num())
return Num()
elif isinstance(expected_intersect, Str):
recur(node.left, Str())
return Str()
else:
recur(node.left, union_type)
return union_type
elif operator == 'Mod':
# num % num OR str % unknown
union_type = Union(Num(), Str())
expected_intersect = type_intersection(expected_type, union_type)
left_intersect = type_intersection(probe(node.left), union_type)
if isinstance(left_intersect, Num):
recur(node.left, Num())
recur(node.right, Num())
return Num()
elif isinstance(left_intersect, Str):
recur(node.left, Str())
recur(node.right, Unknown())
return Str()
elif isinstance(expected_intersect, Num):
recur(node.left, Num())
recur(node.right, Num())
return Num()
elif isinstance(expected_intersect, Str):
recur(node.left, Str())
recur(node.right, Unknown())
return Str()
else:
recur(node.left, union_type)
recur(node.right, Unknown())
return union_type
else:
recur(node.left, Num())
recur(node.right, Num())
return Num()
if token == 'UnaryOp':
if get_token(node.op) == 'Not':
recur(node.operand, Bool())
return Bool()
else:
recur(node.operand, Num())
return Num()
if token == 'Lambda':
return construct_function_type(node, LambdaVisitor(context))
if token == 'IfExp':
recur(node.test, Bool())
if_inferences, else_inferences = maybe_inferences(node.test, context)
context.begin_scope(Scope(if_inferences))
body_type = recur(node.body, expected_type)
context.end_scope()
context.begin_scope(Scope(else_inferences))
else_type = recur(node.orelse, expected_type)
context.end_scope()
return unify_types([body_type, else_type])
if token == 'Dict':
key_type = unify_types([recur(key, Unknown()) for key in node.keys])
value_type = unify_types([recur(value, Unknown())
for value in node.values])
return Dict(key_type, value_type)
if token == 'Set':
subtype = (expected_type.item_type if isinstance(expected_type, Set)
else Unknown())
return Set(unify_types([recur(elt, Unknown()) for elt in node.elts]))
if token == 'ListComp':
subtype = (expected_type.item_type if isinstance(expected_type, List)
else Unknown())
return List(comp(node.elt, node.generators, subtype))
if token == 'SetComp':
subtype = (expected_type.item_type if isinstance(expected_type, Set)
else Unknown())
return Set(comp(node.elt, node.generators, subtype))
if token == 'DictComp':
expected_key_type = (expected_type.key_type
if isinstance(expected_type, Dict)
else Unknown())
expected_value_type = (expected_type.value_type
if isinstance(expected_type, Dict)
else Unknown())
key_type = comp(node.key, node.generators, expected_key_type)
value_type = comp(node.value, node.generators, expected_value_type)
return Dict(key_type, value_type)
if token == 'GeneratorExp':
subtype = (expected_type.item_type if isinstance(expected_type, List)
else Unknown())
return List(comp(node.elt, node.generators, subtype))
if token == 'Yield':
return List(recur(node.value, Unknown()))
if token == 'Compare':
operator = get_token(node.ops[0])
if len(node.ops) > 1 or len(node.comparators) > 1:
warnings.warn(node, 'comparison-operator-chaining')
if operator in ['Eq', 'NotEq', 'Lt', 'LtE', 'Gt', 'GtE']:
# all operands are constrained to have the same type
# as their intersection
left_probe = probe(node.left)
right_probe = probe(node.comparators[0])
intersection = type_intersection(left_probe, right_probe)
if intersection is None:
recur(node.left, right_probe)
recur(node.comparators[0], left_probe)
else:
recur(node.left, intersection)
recur(node.comparators[0], intersection)
if operator in ['Is', 'IsNot']:
recur(node.left, Maybe(Unknown()))
recur(node.comparators[0], NoneType())
if operator in ['In', 'NotIn']:
# constrain right to list/set of left, and left to inst. of right
left_probe = probe(node.left)
right_probe = probe(node.comparators[0])
union_type = Union(List(left_probe), Set(left_probe),
Dict(left_probe, Unknown()), Str())
recur(node.comparators[0], union_type)
if isinstance(right_probe, (List, Set)):
recur(node.left, right_probe.item_type)
elif isinstance(right_probe, Dict):
recur(node.left, right_probe.key_type)
else:
recur(node.left, Unknown())
return Bool()
if token == 'Call':
function_type = recur(node.func, Unknown())
if not isinstance(function_type, (Class, Function)):
if not isinstance(function_type, Unknown):
warnings.warn(node, 'not-a-function')
return Unknown()
signature = function_type.signature
instance = (function_type.instance
if isinstance(function_type, Function) else None)
offset = 1 if (instance is not None
or isinstance(function_type, Class)) else 0
argument_scope = Scope()
if instance is not None:
self_symbol = Symbol(signature.names[0], instance)
argument_scope.add(self_symbol)
# make sure all required arguments are specified
if node.starargs is None and node.kwargs is None:
start = offset + len(node.args)
required = signature.names[start:signature.min_count]
kwarg_names = [keyword.arg for keyword in node.keywords]
missing = [name for name in required if name not in kwarg_names]
for missing_argument in missing:
warnings.warn(node, 'missing-argument', missing_argument)
# check for too many arguments
if signature.vararg_name is None:
if len(node.args) + len(node.keywords) > len(signature.types):
warnings.warn(node, 'too-many-arguments')
# load positional arguments
for i, arg in enumerate(node.args):
if i + offset >= len(signature):
break
arg_type = recur(arg, signature.types[i + offset])
value = static_evaluate(arg, context)
argument_scope.add(Symbol(signature.names[i + offset],
arg_type, value))
# load keyword arguments
for kwarg in node.keywords:
# TODO: make sure there is no overlap with positional args
expected_type = signature.get_dict().get(kwarg.arg)
if expected_type is None:
warnings.warn(node, 'extra-keyword', kwarg.arg)
else:
arg_type = recur(kwarg.value, expected_type)
value = static_evaluate(kwarg.value, context)
argument_scope.add(Symbol(kwarg.arg, arg_type, value))
if node.starargs is not None:
recur(node.starargs, List(Unknown()))
if node.kwargs is not None:
recur(node.kwargs, Dict(Unknown(), Unknown()))
return_type, _ = function_type.evaluator.evaluate(argument_scope)
return return_type
if token == 'Repr':
return Str()
if token == 'Num':
return Num()
if token == 'Str':
return Str()
if token == 'Attribute':
value_type = recur(node.value, Unknown())
if isinstance(value_type, Unknown):
return Unknown()
if not isinstance(value_type, Instance):
warnings.warn(node, 'not-an-instance')
return Unknown()
attr_type = value_type.attributes.get_type(node.attr)
if attr_type is None:
warnings.warn(node, 'not-a-member')
return Unknown()
return attr_type
if token == 'Subscript':
union_type = Union(List(Unknown()), Dict(Unknown(), Unknown()),
BaseTuple())
value_type = recur(node.value, union_type)
if get_token(node.slice) == 'Index':
if isinstance(value_type, Tuple):
index = static_evaluate(node.slice.value, context)
if isinstance(index, UnknownValue):
return Unknown()
if not isinstance(index, int):
return Unknown()
if not 0 <= index < len(value_type.item_types):
return Unknown()
return value_type.item_types[index]
elif isinstance(value_type, List):
return value_type.item_type
elif isinstance(value_type, Dict):
return value_type.value_type
else:
return Unknown()
elif get_token(node.slice) == 'Slice':
if node.slice.lower is not None:
recur(node.slice.lower, Num())
if node.slice.upper is not None:
recur(node.slice.upper, Num())
if node.slice.step is not None:
recur(node.slice.step, Num())
return value_type
else:
return value_type
if token == 'Name':
defined_type = context.get_type(node.id)
if defined_type is None:
warnings.warn(node, 'undefined', node.id)
context.add_constraint(node.id, expected_type)
return defined_type or Unknown()
if token == 'List':
subtype = (expected_type.item_type if isinstance(expected_type, List)
else Unknown())
return List(unify_types([recur(elt, subtype) for elt in node.elts]))
if token == 'Tuple':
if (isinstance(expected_type, Tuple)
and len(node.elts) == len(expected_type.item_types)):
return Tuple([recur(element, type_) for element, type_ in
zip(node.elts, expected_type.item_types)])
return Tuple([recur(element, Unknown()) for element in node.elts])
raise Exception('visit_expression does not recognize ' + token)