def convert_leaf(self, grammar, type, value, prefix, start_pos):
#print('leaf', value, pytree.type_repr(type))
if type == tokenize.NAME:
if value in grammar.keywords:
if value in ('def', 'class', 'lambda'):
self._scope_names_stack.append({})
return pt.Keyword(self.position_modifier, value, start_pos,
prefix)
else:
name = pt.Name(self.position_modifier, value, start_pos,
prefix)
# Keep a listing of all used names
arr = self._used_names.setdefault(name.value, [])
arr.append(name)
arr = self._scope_names_stack[-1].setdefault(name.value, [])
arr.append(name)
return name
elif type == STRING:
return pt.String(self.position_modifier, value, start_pos, prefix)
elif type == NUMBER:
return pt.Number(self.position_modifier, value, start_pos, prefix)
elif type in (NEWLINE, ENDMARKER):
return pt.Whitespace(self.position_modifier, value, start_pos,
prefix)
else:
return pt.Operator(self.position_modifier, value, start_pos,
prefix)
def convert_leaf(self, grammar, type, value, prefix, start_pos):
# print('leaf', repr(value), token.tok_name[type])
if type == tokenize.NAME:
if value in grammar.keywords:
if value in ('def', 'class', 'lambda'):
self._scope_names_stack.append({})
return pt.Keyword(self.position_modifier, value, start_pos,
prefix)
else:
name = pt.Name(self.position_modifier, value, start_pos,
prefix)
# Keep a listing of all used names
arr = self._used_names.setdefault(name.value, [])
arr.append(name)
arr = self._scope_names_stack[-1].setdefault(name.value, [])
arr.append(name)
return name
elif type == STRING:
return pt.String(self.position_modifier, value, start_pos, prefix)
elif type == NUMBER:
return pt.Number(self.position_modifier, value, start_pos, prefix)
elif type == NEWLINE:
return pt.Newline(self.position_modifier, value, start_pos, prefix)
elif type == INDENT:
return pt.Indent(self.position_modifier, value, start_pos, prefix)
elif type == DEDENT:
return pt.Dedent(self.position_modifier, value, start_pos, prefix)
elif type == ENDMARKER:
return pt.EndMarker(self.position_modifier, value, start_pos,
prefix)
else:
return pt.Operator(self.position_modifier, value, start_pos,
prefix)
def convert_leaf(self, grammar, type, value, prefix, start_pos):
# print('leaf', repr(value), token.tok_name[type])
if type == tokenize.NAME:
if value in grammar.keywords:
return pt.Keyword(value, start_pos, prefix)
else:
name = pt.Name(value, start_pos, prefix)
# Keep a listing of all used names
arr = self._used_names.setdefault(name.value, [])
arr.append(name)
return name
elif type == STRING:
return pt.String(value, start_pos, prefix)
elif type == NUMBER:
return pt.Number(value, start_pos, prefix)
elif type == NEWLINE:
return pt.Newline(value, start_pos, prefix)
elif type == ENDMARKER:
return pt.EndMarker(value, start_pos, prefix)
else:
return pt.Operator(value, start_pos, prefix)
def parent(self):
"""
Creating fake statements for the interpreter.
"""
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# TODO this import is wrong. Yields x for x.y.z instead of z
module = __import__(module_name)
parser_path = names
raw_module = get_module(self._value)
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
path = re.sub('c$', '', path)
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path[:-1]).module
if parser_path:
assert len(parser_path) == 1
found = self._evaluator.find_types(mod,
parser_path[0],
search_global=True)
else:
found = [er.wrap(self._evaluator, mod)]
if not found:
debug.warning(
'Possibly an interpreter lookup for Python code failed %s',
parser_path)
if not found:
evaluated = compiled.CompiledObject(obj)
if evaluated == builtins:
# The builtins module is special and always cached.
evaluated = compiled.builtin
found = [evaluated]
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content
])
stmt.parent = self._module
return stmt
def parent(self):
"""
Creating fake statements for the interpreter.
Here we are trying to link back to Python code, if possible. This means
we try to find the python module for a name (not the builtin).
"""
return mixed.create(self._evaluator, self._value)
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# If we put anything into fromlist, it will just return the
# latest name.
module = __import__(module_name, fromlist=[''])
parser_path = names
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
# Find the corresponding Python file for an interpreted one.
path = re.sub(r'\.pyc$', '.py', path)
if path.endswith('.py'):
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path).module
mod = self._evaluator.wrap(mod)
# We have to make sure that the modules that we import are also
# part of evaluator.modules.
for module_name, module in sys.modules.items():
try:
iterated_path = module.__file__
except AttributeError:
pass
else:
if iterated_path == path:
self._evaluator.modules[module_name] = mod
break
else:
raise NotImplementedError(
'This should not happen, a module '
'should be part of sys.modules.')
if parser_path:
assert len(parser_path) == 1
found = list(
self._evaluator.find_types(mod,
parser_path[0],
search_global=True))
else:
found = [mod]
if not found:
debug.warning(
'Interpreter lookup failed in global scope for %s',
parser_path)
if not found:
evaluated = compiled.create(self._evaluator, obj)
found = [evaluated]
if len(found) > 1:
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''),
content
])
stmt.parent = self._module
return stmt
else:
return found[0]