def _six_fail_hook(modname):
"""Fix six.moves imports due to the dynamic nature of this
class.
Construct a pseudo-module which contains all the necessary imports
for six
:param modname: Name of failed module
:type modname: str
:return: An astroid module
:rtype: nodes.Module
"""
attribute_of = (modname != "six.moves" and
modname.startswith("six.moves"))
if modname != 'six.moves' and not attribute_of:
raise AstroidBuildingError(modname=modname)
module = AstroidBuilder(MANAGER).string_build(_IMPORTS)
module.name = 'six.moves'
if attribute_of:
# Facilitate import of submodules in Moves
start_index = len(module.name)
attribute = modname[start_index:].lstrip(".").replace(".", "_")
try:
import_attr = module.getattr(attribute)[0]
except AttributeInferenceError:
raise AstroidBuildingError(modname=modname)
if isinstance(import_attr, nodes.Import):
submodule = MANAGER.ast_from_module_name(import_attr.names[0][0])
return submodule
# Let dummy submodule imports pass through
# This will cause an Uninferable result, which is okay
return module
def test_nameconstant(self):
# used to fail for Python 3.4
builder = AstroidBuilder()
astroid = builder.string_build("def test(x=True): pass")
default = astroid.body[0].args.args[0]
self.assertEqual(default.name, 'x')
self.assertEqual(next(default.infer()).value, True)
def six_moves_transform():
code = dedent('''
class Moves(object):
{}
moves = Moves()
''').format(_indent(_IMPORTS, " "))
module = AstroidBuilder(MANAGER).string_build(code)
module.name = 'six.moves'
return module
def infer_enum_class(node):
""" Specific inference for enums. """
for basename in node.basenames:
# TODO: doesn't handle subclasses yet. This implementation
# is a hack to support enums.
if basename not in ENUM_BASE_NAMES:
continue
if node.root().name == "enum":
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssignName) for value in values):
continue
targets = []
stmt = values[0].statement()
if isinstance(stmt, nodes.Assign):
if isinstance(stmt.targets[0], nodes.Tuple):
targets = stmt.targets[0].itered()
else:
targets = stmt.targets
elif isinstance(stmt, nodes.AnnAssign):
targets = [stmt.target]
inferred_return_value = None
if isinstance(stmt.value, nodes.Const):
if isinstance(stmt.value.value, str):
inferred_return_value = '"{}"'.format(stmt.value.value)
else:
inferred_return_value = stmt.value.value
new_targets = []
for target in targets:
# Replace all the assignments with our mocked class.
classdef = dedent(
"""
class {name}({types}):
@property
def value(self):
return {return_value}
@property
def name(self):
return {name}
""".format(
name=target.name,
types=", ".join(node.basenames),
return_value=inferred_return_value,
)
)
fake = AstroidBuilder(MANAGER).string_build(classdef)[target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instantiate_class())
node.locals[local] = new_targets
break
return node
def rebuild_ast(tree, modname='', package=False):
manager = AstroidManager()
builder = AstroidBuilder(manager)
rebuilder = TreeRebuilder(manager)
module = rebuilder.visit_module(tree, modname=modname, package=package)
module._from_nodes = rebuilder._from_nodes
module._delayed_assattr = rebuilder._delayed_assattr
module = builder._post_build(module, 'utf8')
return module
def test_with_infer_assnames(self):
builder = AstroidBuilder()
data = """
with open('a.txt') as stream, open('b.txt'):
stream.read()
"""
astroid = builder.string_build(data, __name__, __file__)
# Used to crash due to the fact that the second
# context manager didn't use an assignment name.
list(astroid.nodes_of_class(nodes.CallFunc))[-1].infered()
def _extend_str(class_node, rvalue):
"""function to extend builtin str/unicode class"""
code = dedent(
"""
class whatever(object):
def join(self, iterable):
return {rvalue}
def replace(self, old, new, count=None):
return {rvalue}
def format(self, *args, **kwargs):
return {rvalue}
def encode(self, encoding='ascii', errors=None):
return ''
def decode(self, encoding='ascii', errors=None):
return u''
def capitalize(self):
return {rvalue}
def title(self):
return {rvalue}
def lower(self):
return {rvalue}
def upper(self):
return {rvalue}
def swapcase(self):
return {rvalue}
def index(self, sub, start=None, end=None):
return 0
def find(self, sub, start=None, end=None):
return 0
def count(self, sub, start=None, end=None):
return 0
def strip(self, chars=None):
return {rvalue}
def lstrip(self, chars=None):
return {rvalue}
def rstrip(self, chars=None):
return {rvalue}
def rjust(self, width, fillchar=None):
return {rvalue}
def center(self, width, fillchar=None):
return {rvalue}
def ljust(self, width, fillchar=None):
return {rvalue}
"""
)
code = code.format(rvalue=rvalue)
fake = AstroidBuilder(MANAGER).string_build(code)["whatever"]
for method in fake.mymethods():
method.parent = class_node
method.lineno = None
method.col_offset = None
if "__class__" in method.locals:
method.locals["__class__"] = [class_node]
class_node.locals[method.name] = [method]
method.parent = class_node
def test_io_is__io(self):
# _io module calls itself io. This leads
# to cyclic dependencies when astroid tries to resolve
# what io.BufferedReader is. The code that handles this
# is in astroid.raw_building.imported_member, which verifies
# the true name of the module.
import _io
builder = AstroidBuilder()
module = builder.inspect_build(_io)
buffered_reader = module.getattr('BufferedReader')[0]
self.assertEqual(buffered_reader.root().name, 'io')
def redbaron_transform(module):
""" Let pylint know the redbaron module """
if module.name != 'redbaron':
return
astroid_mgr = AstroidBuilder(MANAGER)
fake = astroid_mgr.string_build(PYTEST_STUB)
for stub in ('NameNode', 'PassNode'):
text = NODE_PLACEHOLDER_CLASS_STUB.format(name=stub)
fake = astroid_mgr.string_build(text)
module.locals[stub] = fake.locals[stub]
def _extend_str(class_node, rvalue):
"""function to extend builtin str/unicode class"""
# TODO(cpopa): this approach will make astroid to believe
# that some arguments can be passed by keyword, but
# unfortunately, strings and bytes don't accept keyword arguments.
code = dedent('''
class whatever(object):
def join(self, iterable):
return {rvalue}
def replace(self, old, new, count=None):
return {rvalue}
def format(self, *args, **kwargs):
return {rvalue}
def encode(self, encoding='ascii', errors=None):
return ''
def decode(self, encoding='ascii', errors=None):
return u''
def capitalize(self):
return {rvalue}
def title(self):
return {rvalue}
def lower(self):
return {rvalue}
def upper(self):
return {rvalue}
def swapcase(self):
return {rvalue}
def index(self, sub, start=None, end=None):
return 0
def find(self, sub, start=None, end=None):
return 0
def count(self, sub, start=None, end=None):
return 0
def strip(self, chars=None):
return {rvalue}
def lstrip(self, chars=None):
return {rvalue}
def rstrip(self, chars=None):
return {rvalue}
def rjust(self, width, fillchar=None):
return {rvalue}
def center(self, width, fillchar=None):
return {rvalue}
def ljust(self, width, fillchar=None):
return {rvalue}
''')
code = code.format(rvalue=rvalue)
fake = AstroidBuilder(MANAGER).string_build(code)['whatever']
for method in fake.mymethods():
class_node.locals[method.name] = [method]
method.parent = class_node
def transform(cls):
if cls.name == 'scoped_session':
builder = AstroidBuilder(MANAGER)
module_node = builder.module_build(sys.modules[Session.__module__])
session_cls_node = [
c for c in module_node.get_children()
if getattr(c, "type", None) == "class" and c.name == Session.__name__
][0]
for prop in Session.public_methods:
cls.locals[prop] = [
c for c in session_cls_node.get_children()
if getattr(c, "type", None) == "method" and c.name == prop
]
def pytest_transform(module):
""" Let pylint know the pytest module """
pytest = ('pytest', 'py.test')
if module.name not in pytest:
return
astroid_mgr = AstroidBuilder(MANAGER)
fake = astroid_mgr.string_build(PYTEST_STUB)
for complex_stub in ('mark',):
module.locals[complex_stub] = fake.locals[complex_stub]
for stub in ('skip', 'xfail', 'fixture', 'test', 'raises'):
text = PLACEHOLDER_CLASS_STUB.format(name=stub)
fake = astroid_mgr.string_build(text)
module.locals[stub] = fake.locals[stub]
def test_filter_stmts_scoping(self):
builder = AstroidBuilder()
data = """
def test():
compiler = int()
class B(compiler.__class__):
pass
compiler = B()
return compiler
"""
astroid = builder.string_build(data, __name__, __file__)
test = astroid['test']
result = next(test.infer_call_result(astroid))
self.assertIsInstance(result, Instance)
base = next(result._proxied.bases[0].infer())
self.assertEqual(base.name, 'int')
def __init__(self, type_constraints) -> None:
"""Initialize a type store with all the built-in types from the typeshed module."""
self.type_constraints = type_constraints
self.classes = defaultdict(lambda: defaultdict(list))
self.functions = defaultdict(list)
self.methods = defaultdict(list)
builder = AstroidBuilder()
module = builder.file_build(TYPE_SHED_PATH)
self._parse_classes(module)
self._parse_functions(module)
# Add in initializers
for klass_name, methods in self.classes.items():
if '__init__' in methods:
self.functions[klass_name] = [class_callable(init) for init, _ in methods['__init__']]
def set_source_text(text):
global _reporter
global _linter
assert isinstance(_reporter, Reporter)
assert isinstance(_linter, lint.PyLinter)
_reporter.reset()
builder = AstroidBuilder()
try:
astroid_module = builder.string_build(text)
except:
return
_linter.set_current_module("<stdin>")
_linter.check_astroid_module(astroid_module, _walker, _raw_checkers, _token_checkers)
assert isinstance(text, str)
_reporter.improve_errors_positions(text)
def infer_enum_class(node):
""" Specific inference for enums. """
names = set(('Enum', 'IntEnum', 'enum.Enum', 'enum.IntEnum'))
for basename in node.basenames:
# TODO: doesn't handle subclasses yet. This implementation
# is a hack to support enums.
if basename not in names:
continue
if node.root().name == 'enum':
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssignName)
for value in values):
continue
stmt = values[0].statement()
if isinstance(stmt, nodes.Assign):
if isinstance(stmt.targets[0], nodes.Tuple):
targets = stmt.targets[0].itered()
else:
targets = stmt.targets
elif isinstance(stmt, nodes.AnnAssign):
targets = [stmt.target]
new_targets = []
for target in targets:
# Replace all the assignments with our mocked class.
classdef = dedent('''
class %(name)s(%(types)s):
@property
def value(self):
# Not the best return.
return None
@property
def name(self):
return %(name)r
''' % {'name': target.name, 'types': ', '.join(node.basenames)})
fake = AstroidBuilder(MANAGER).string_build(classdef)[target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instantiate_class())
node.locals[local] = new_targets
break
return node
def test_decorator_callchain_issue42(self):
builder = AstroidBuilder()
data = """
def test():
def factory(func):
def newfunc():
func()
return newfunc
return factory
@test()
def crash():
pass
"""
astroid = builder.string_build(data, __name__, __file__)
self.assertEqual(astroid['crash'].type, 'function')
def test_numpy_crash(self):
"""test don't crash on numpy"""
#a crash occured somewhere in the past, and an
# InferenceError instead of a crash was better, but now we even infer!
try:
import numpy # pylint: disable=unused-variable
except ImportError:
self.skipTest('test skipped: numpy is not available')
builder = AstroidBuilder()
data = """
from numpy import multiply
multiply(1, 2, 3)
"""
astroid = builder.string_build(data, __name__, __file__)
callfunc = astroid.body[1].value.func
inferred = callfunc.inferred()
self.assertEqual(len(inferred), 1)
def test_new_style_class_detection(self):
try:
import pygtk
except ImportError:
self.skipTest('test skipped: pygtk is not available')
# XXX may fail on some pygtk version, because objects in
# gobject._gobject have __module__ set to gobject :(
builder = AstroidBuilder()
data = """
import pygtk
pygtk.require("2.6")
import gobject
class A(gobject.GObject):
pass
"""
astroid = builder.string_build(data, __name__, __file__)
a = astroid['A']
self.assertTrue(a.newstyle)
def _settings_transform():
module = AstroidBuilder(AstroidManager()).string_build(
textwrap.dedent("""
class Settings(object):
os = None
arch = None
compiler = None
build_type = None
""")
)
return module['Settings']
def infer_named_tuple(node, context=None):
"""Specific inference function for namedtuple Call node"""
class_node, name, attributes = infer_func_form(node,
nodes.Tuple._proxied,
context=context)
call_site = arguments.CallSite.from_call(node)
func = next(
extract_node('import collections; collections.namedtuple').infer())
try:
rename = next(call_site.infer_argument(func, 'rename',
context)).bool_value()
except InferenceError:
rename = False
if rename:
attributes = _get_renamed_namedtuple_atributes(attributes)
replace_args = ', '.join('{arg}=None'.format(arg=arg)
for arg in attributes)
field_def = (" {name} = property(lambda self: self[{index:d}], "
"doc='Alias for field number {index:d}')")
field_defs = '\n'.join(
field_def.format(name=name, index=index)
for index, name in enumerate(attributes))
fake = AstroidBuilder(MANAGER).string_build(
'''
class %(name)s(tuple):
__slots__ = ()
_fields = %(fields)r
def _asdict(self):
return self.__dict__
@classmethod
def _make(cls, iterable, new=tuple.__new__, len=len):
return new(cls, iterable)
def _replace(self, %(replace_args)s):
return self
def __getnewargs__(self):
return tuple(self)
%(field_defs)s
''' % {
'name': name,
'fields': attributes,
'field_defs': field_defs,
'replace_args': replace_args
})
class_node.locals['_asdict'] = fake.body[0].locals['_asdict']
class_node.locals['_make'] = fake.body[0].locals['_make']
class_node.locals['_replace'] = fake.body[0].locals['_replace']
class_node.locals['_fields'] = fake.body[0].locals['_fields']
for attr in attributes:
class_node.locals[attr] = fake.body[0].locals[attr]
# we use UseInferenceDefault, we can't be a generator so return an iterator
return iter([class_node])
def test_recursion_regression_issue25(self):
builder = AstroidBuilder()
data = """
import recursion as base
_real_Base = base.Base
class Derived(_real_Base):
pass
def run():
base.Base = Derived
"""
astroid = builder.string_build(data, __name__, __file__)
# Used to crash in _is_metaclass, due to wrong
# ancestors chain
classes = astroid.nodes_of_class(nodes.Class)
for klass in classes:
# triggers the _is_metaclass call
klass.type
def thread_transform():
return AstroidBuilder(MANAGER).string_build('''
class lock(object):
def acquire(self, blocking=True):
pass
def release(self):
pass
def Lock():
return lock()
''')
def test_recursion_regression_issue25(self):
builder = AstroidBuilder()
data = """
import recursion as base
_real_Base = base.Base
class Derived(_real_Base):
pass
def run():
base.Base = Derived
"""
astroid = builder.string_build(data, __name__, __file__)
# Used to crash in _is_metaclass, due to wrong
# ancestors chain
classes = astroid.nodes_of_class(nodes.Class)
for klass in classes:
# triggers the _is_metaclass call
klass.type
def ipaplatform_services_transform():
return AstroidBuilder(MANAGER).string_build(
textwrap.dedent('''
from ipaplatform.base.services import knownservices
from ipaplatform.base.services import timedate_services
from ipaplatform.base.services import service
from ipaplatform.base.services import wellknownservices
from ipaplatform.base.services import wellknownports
__all__ = ('knownservices', 'timedate_services', 'service',
'wellknownservices', 'wellknownports')
'''))
def infer_named_tuple(node, context=None):
"""Specific inference function for namedtuple Call node"""
tuple_base_name = nodes.Name(name="tuple", parent=node.root())
class_node, name, attributes = infer_func_form(node,
tuple_base_name,
context=context)
call_site = arguments.CallSite.from_call(node, context=context)
func = next(
extract_node("import collections; collections.namedtuple").infer())
try:
rename = next(call_site.infer_argument(func, "rename",
context)).bool_value()
except InferenceError:
rename = False
if rename:
attributes = _get_renamed_namedtuple_attributes(attributes)
replace_args = ", ".join("{arg}=None".format(arg=arg)
for arg in attributes)
field_def = (" {name} = property(lambda self: self[{index:d}], "
"doc='Alias for field number {index:d}')")
field_defs = "\n".join(
field_def.format(name=name, index=index)
for index, name in enumerate(attributes))
fake = AstroidBuilder(MANAGER).string_build(
"""
class %(name)s(tuple):
__slots__ = ()
_fields = %(fields)r
def _asdict(self):
return self.__dict__
@classmethod
def _make(cls, iterable, new=tuple.__new__, len=len):
return new(cls, iterable)
def _replace(self, %(replace_args)s):
return self
def __getnewargs__(self):
return tuple(self)
%(field_defs)s
""" % {
"name": name,
"fields": attributes,
"field_defs": field_defs,
"replace_args": replace_args,
})
class_node.locals["_asdict"] = fake.body[0].locals["_asdict"]
class_node.locals["_make"] = fake.body[0].locals["_make"]
class_node.locals["_replace"] = fake.body[0].locals["_replace"]
class_node.locals["_fields"] = fake.body[0].locals["_fields"]
for attr in attributes:
class_node.locals[attr] = fake.body[0].locals[attr]
# we use UseInferenceDefault, we can't be a generator so return an iterator
return iter([class_node])
def transform(module):
'''Add fake locals to a module's namespace.'''
# Generate the path to the fake module
dirname = os.path.dirname(__file__)
path = os.path.join(dirname, module.name + '.py')
if not os.path.exists(path):
return
# If the file exists, add fakes to the module's namespace
fake = AstroidBuilder(MANAGER).file_build(path)
for name, obj in fake.locals.iteritems():
module.locals.setdefault(name, []).extend(obj)
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in ('.zip', '.egg'):
try:
eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace(os.path.sep, '.')
if importer.is_package(resource):
zmodname = zmodname + '.__init__'
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except:
continue
return None
def mechanize_transform():
return AstroidBuilder(MANAGER).string_build("""
class Browser(object):
def open(self, url, data=None, timeout=None):
return None
def open_novisit(self, url, data=None, timeout=None):
return None
def open_local_file(self, filename):
return None
""")
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in ('.zip', '.egg'):
try:
eggpath, resource = filepath.rsplit(ext + '/', 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace('/', '.')
if importer.is_package(resource):
zmodname = zmodname + '.__init__'
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except:
continue
return None
def handle_session(cls):
"""Handle the sqlalchemy.orm.scoping.scoped_session."""
if cls.name == 'scoped_session':
source_files = cls.parent.source_file.split('/')
source_files[-1] = 'session.py'
module = AstroidBuilder(MANAGER).file_build('/'.join(source_files),
'sqlalchemy.orm.session')
session = module.locals.get('Session')[0]
for method in Session.public_methods:
cls.locals[method] = session.locals[method]
return cls
def infer_enum_class(node, context=None):
""" Specific inference for enums. """
names = set(('Enum', 'IntEnum', 'enum.Enum', 'enum.IntEnum'))
for basename in node.basenames:
# TODO: doesn't handle subclasses yet.
if basename not in names:
continue
if node.root().name == 'enum':
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssName) for value in values):
continue
parent = values[0].parent
real_value = parent.value
new_targets = []
for target in parent.targets:
# Replace all the assignments with our mocked class.
classdef = dedent('''
class %(name)s(object):
@property
def value(self):
return %(value)s
@property
def name(self):
return %(name)r
%(name)s = %(value)s
''' % {
'name': target.name,
'value': real_value.as_string()
})
fake = AstroidBuilder(MANAGER).string_build(classdef)[
target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instanciate_class())
node.locals[local] = new_targets
break
return node
def subprocess_transform():
if PY3K:
communicate = (bytes('string', 'ascii'), bytes('string', 'ascii'))
init = """
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, restore_signals=True,
start_new_session=False, pass_fds=()):
pass
"""
else:
communicate = ('string', 'string')
init = """
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
pass
"""
if PY33:
wait_signature = 'def wait(self, timeout=None)'
else:
wait_signature = 'def wait(self)'
return AstroidBuilder(MANAGER).string_build(
'''
class Popen(object):
returncode = pid = 0
stdin = stdout = stderr = file()
%(init)s
def communicate(self, input=None):
return %(communicate)r
%(wait_signature)s:
return self.returncode
def poll(self):
return self.returncode
def send_signal(self, signal):
pass
def terminate(self):
pass
def kill(self):
pass
''' % {
'init': init,
'communicate': communicate,
'wait_signature': wait_signature
})
def _import_gi_module(modname):
# we only consider gi.repository submodules
if not modname.startswith("gi.repository."):
raise AstroidBuildingError(modname=modname)
# build astroid representation unless we already tried so
if modname not in _inspected_modules:
modnames = [modname]
optional_modnames = []
# GLib and GObject may have some special case handling
# in pygobject that we need to cope with. However at
# least as of pygobject3-3.13.91 the _glib module doesn't
# exist anymore, so if treat these modules as optional.
if modname == "gi.repository.GLib":
optional_modnames.append("gi._glib")
elif modname == "gi.repository.GObject":
optional_modnames.append("gi._gobject")
try:
modcode = ""
for m in itertools.chain(modnames, optional_modnames):
try:
with warnings.catch_warnings():
# Just inspecting the code can raise gi deprecation
# warnings, so ignore them.
try:
from gi import ( # pylint:disable=import-error
PyGIDeprecationWarning, PyGIWarning,
)
warnings.simplefilter("ignore",
PyGIDeprecationWarning)
warnings.simplefilter("ignore", PyGIWarning)
except Exception: # pylint:disable=broad-except
pass
__import__(m)
modcode += _gi_build_stub(sys.modules[m])
except ImportError:
if m not in optional_modnames:
raise
except ImportError:
astng = _inspected_modules[modname] = None
else:
astng = AstroidBuilder(AstroidManager()).string_build(
modcode, modname)
_inspected_modules[modname] = astng
else:
astng = _inspected_modules[modname]
if astng is None:
raise AstroidBuildingError(modname=modname)
return astng
def pkg_resources_transform():
return AstroidBuilder(MANAGER).string_build('''
def require(*requirements):
return pkg_resources.working_set.require(*requirements)
def run_script(requires, script_name):
return pkg_resources.working_set.run_script(requires, script_name)
def iter_entry_points(group, name=None):
return pkg_resources.working_set.iter_entry_points(group, name)
def resource_exists(package_or_requirement, resource_name):
return get_provider(package_or_requirement).has_resource(resource_name)
def resource_isdir(package_or_requirement, resource_name):
return get_provider(package_or_requirement).resource_isdir(
resource_name)
def resource_filename(package_or_requirement, resource_name):
return get_provider(package_or_requirement).get_resource_filename(
self, resource_name)
def resource_stream(package_or_requirement, resource_name):
return get_provider(package_or_requirement).get_resource_stream(
self, resource_name)
def resource_string(package_or_requirement, resource_name):
return get_provider(package_or_requirement).get_resource_string(
self, resource_name)
def resource_listdir(package_or_requirement, resource_name):
return get_provider(package_or_requirement).resource_listdir(
resource_name)
def extraction_error():
pass
def get_cache_path(archive_name, names=()):
extract_path = self.extraction_path or get_default_cache()
target_path = os.path.join(extract_path, archive_name+'-tmp', *names)
return target_path
def postprocess(tempname, filename):
pass
def set_extraction_path(path):
pass
def cleanup_resources(force=False):
pass
''')
def transform(module):
if module.name == 'numpy':
fake = AstroidBuilder(manager=MANAGER).string_build('''
import numpy
class array(numpy.ndarray):
pass
class recarray(numpy.ndarray):
pass
''')
for name in ("array", "recarray"):
module.locals[name] = fake.locals[name]
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in (".zip", ".egg"):
try:
eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace(os.path.sep, ".")
if importer.is_package(resource):
zmodname = zmodname + ".__init__"
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except Exception: # pylint: disable=broad-except
continue
return None
def nose_transform():
"""Custom transform for the nose.tools module."""
builder = AstroidBuilder(MANAGER)
stub = AstroidBuilder(MANAGER).string_build('''__all__ = []''')
unittest_module = builder.module_build(unittest.case)
case = unittest_module['TestCase']
all_entries = ['ok_', 'eq_']
for method_name, method in case.locals.items():
if method_name.startswith('assert') and '_' not in method_name:
pep8_name = _pep8(method_name)
all_entries.append(pep8_name)
stub[pep8_name] = method[0]
# Update the __all__ variable, since nose.tools
# does this manually with .append.
all_assign = stub['__all__'].parent
all_object = List(all_entries)
all_object.parent = all_assign
all_assign.value = all_object
return stub
def infer_enum_class(node, context=None):
""" Specific inference for enums. """
names = set(('Enum', 'IntEnum', 'enum.Enum', 'enum.IntEnum'))
for basename in node.basenames:
# TODO: doesn't handle subclasses yet.
if basename not in names:
continue
if node.root().name == 'enum':
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssName)
for value in values):
continue
parent = values[0].parent
real_value = parent.value
new_targets = []
for target in parent.targets:
# Replace all the assignments with our mocked class.
classdef = dedent('''
class %(name)s(object):
@property
def value(self):
return %(value)s
@property
def name(self):
return %(name)r
%(name)s = %(value)s
''' % {'name': target.name,
'value': real_value.as_string()})
fake = AstroidBuilder(MANAGER).string_build(classdef)[target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instanciate_class())
node.locals[local] = new_targets
break
return node
class Python3TC(TestCase):
def setUp(self):
self.manager = AstroidManager()
self.builder = AstroidBuilder(self.manager)
self.manager.astroid_cache.clear()
@require_version('3.0')
def test_starred_notation(self):
astroid = self.builder.string_build("*a, b = [1, 2, 3]", 'test', 'test')
# Get the star node
node = next(next(next(astroid.get_children()).get_children()).get_children())
self.assertTrue(isinstance(node.ass_type(), Assign))
def ast_from_module(self, module, modname=None):
"""given an imported module, return the astroid object"""
modname = modname or module.__name__
if modname in self.astroid_cache:
return self.astroid_cache[modname]
try:
# some builtin modules don't have __file__ attribute
filepath = module.__file__
if modutils.is_python_source(filepath):
return self.ast_from_file(filepath, modname)
except AttributeError:
pass
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).module_build(module, modname)
def multiprocessing_transform():
module = AstroidBuilder(MANAGER).string_build(
dedent('''
from multiprocessing.managers import SyncManager
def Manager():
return SyncManager()
'''))
if not PY34:
return module
# On Python 3.4, multiprocessing uses a getattr lookup inside contexts,
# in order to get the attributes they need. Since it's extremely
# dynamic, we use this approach to fake it.
node = AstroidBuilder(MANAGER).string_build(
dedent('''
from multiprocessing.context import DefaultContext, BaseContext
default = DefaultContext()
base = BaseContext()
'''))
try:
context = next(node['default'].infer())
base = next(node['base'].infer())
except InferenceError:
return module
for node in (context, base):
for key, value in node.locals.items():
if key.startswith("_"):
continue
value = value[0]
if isinstance(value, nodes.FunctionDef):
# We need to rebound this, since otherwise
# it will have an extra argument (self).
value = BoundMethod(value, node)
module[key] = value
return module
def _buildAstroidModuleFromComponentModuleName(modname):
from Products.ERP5.ERP5Site import getSite
from Acquisition import aq_base
portal = getSite()
component_tool = aq_base(portal.portal_components)
component_obj = None
component_id = modname[len('erp5.component.'):]
if '_version' in modname:
try:
obj = getattr(component_tool,
component_id.replace('_version', '', 1))
except AttributeError:
raise AstroidBuildingException()
if obj.getValidationState() in ('modified', 'validated'):
component_obj = obj
else:
raise AstroidBuildingException()
else:
try:
package, reference = component_id.split('.', 1)
except ValueError:
raise AstroidBuildingException()
for version in portal.getVersionPriorityNameList():
try:
obj = getattr(component_tool,
'%s.%s.%s' % (package, version, reference))
except AttributeError:
continue
if obj.getValidationState() in ('modified', 'validated'):
version_modname = 'erp5.component.%s.%s_version.%s' % (
package, version, reference)
module = MANAGER.astroid_cache.get(
version_modname,
_buildAstroidModuleFromComponentModuleName(
version_modname))
MANAGER.astroid_cache[modname] = module
return module
if component_obj is None:
raise AstroidBuildingException()
# module_build() could also be used but this requires importing
# the ZODB Component and also monkey-patch it to support PEP-302
# for __file__ starting with '<'
module = AstroidBuilder(MANAGER).string_build(
component_obj.getTextContent(validated_only=True), modname)
return module
def handle_session(cls):
"""Handle the sqlalchemy.orm.scoping.scoped_session."""
if cls.name == "scoped_session":
source_files = cls.parent.file.split("/")
source_files[-1] = "session.py"
module = AstroidBuilder(MANAGER).file_build(
"/".join(source_files), "sqlalchemy.orm.session"
)
session = module.locals.get("Session")[0]
for method in Session.public_methods:
cls.locals[method] = session.locals[method]
return cls
return None
def multiprocessing_managers_transform():
return AstroidBuilder(MANAGER).string_build(
dedent('''
import array
import threading
import multiprocessing.pool as pool
import six
class Namespace(object):
pass
class Value(object):
def __init__(self, typecode, value, lock=True):
self._typecode = typecode
self._value = value
def get(self):
return self._value
def set(self, value):
self._value = value
def __repr__(self):
return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
value = property(get, set)
def Array(typecode, sequence, lock=True):
return array.array(typecode, sequence)
class SyncManager(object):
Queue = JoinableQueue = six.moves.queue.Queue
Event = threading.Event
RLock = threading.RLock
BoundedSemaphore = threading.BoundedSemaphore
Condition = threading.Condition
Barrier = threading.Barrier
Pool = pool.Pool
list = list
dict = dict
Value = Value
Array = Array
Namespace = Namespace
__enter__ = lambda self: self
__exit__ = lambda *args: args
def start(self, initializer=None, initargs=None):
pass
def shutdown(self):
pass
'''))
def cubicweb_transform(module):
# handle objectify_predicate decorator (and its former name until bw compat
# is kept). Only look at module level functions, should be enough.
for assnodes in module.locals.values():
for node in assnodes:
if isinstance(node, FunctionDef) and node.decorators:
for decorator in node.decorators.nodes:
try:
for infered in decorator.infer():
if infered.name in ('objectify_predicate',
'objectify_selector'):
turn_function_to_class(node)
break
else:
continue
break
except InferenceError:
continue
# add yams base types into 'yams.buildobjs', astng doesn't grasp globals()
# magic in there
if module.name == 'yams.buildobjs':
from yams import BASE_TYPES
for etype in BASE_TYPES:
module.locals[etype] = [ClassDef(etype, None)]
# add data() to uiprops module
elif module.name.split('.')[-1] == 'uiprops':
fake = AstroidBuilder(MANAGER).string_build('''
def data(string):
return u''
''')
module.locals['data'] = fake.locals['data']
# handle lower case with underscores for relation names in schema.py
if not module.qname().endswith('.schema'):
return
schema_locals = module.locals
for assnodes in schema_locals.values():
for node in assnodes:
if not isinstance(node, ClassDef):
continue
# XXX can we infer ancestor classes? it would be better to know for sure that
# one of the mother classes is yams.buildobjs.RelationDefinition for instance
for base in node.basenames:
if base in ('RelationDefinition', 'ComputedRelation',
'RelationType'):
new_name = node.name.replace('_', '').capitalize()
schema_locals[new_name] = schema_locals[node.name]
del schema_locals[node.name]
node.name = new_name
def _import_gi_module(modname):
# we only consider gi.repository submodules
if not modname.startswith('gi.repository.'):
raise AstroidBuildingException()
# build astroid representation unless we already tried so
if modname not in _inspected_modules:
modnames = [modname]
optional_modnames = []
# GLib and GObject may have some special case handling
# in pygobject that we need to cope with. However at
# least as of pygobject3-3.13.91 the _glib module doesn't
# exist anymore, so if treat these modules as optional.
if modname == 'gi.repository.GLib':
optional_modnames.append('gi._glib')
elif modname == 'gi.repository.GObject':
optional_modnames.append('gi._gobject')
try:
modcode = ''
for m in itertools.chain(modnames, optional_modnames):
try:
__import__(m)
with warnings.catch_warnings():
# Just inspecting the code can raise gi deprecation
# warnings, so ignore them.
try:
from gi import PyGIDeprecationWarning
warnings.simplefilter("ignore",
PyGIDeprecationWarning)
except Exception:
pass
modcode += _gi_build_stub(sys.modules[m])
except ImportError:
if m not in optional_modnames:
raise
except ImportError:
astng = _inspected_modules[modname] = None
else:
astng = AstroidBuilder(MANAGER).string_build(modcode, modname)
_inspected_modules[modname] = astng
else:
astng = _inspected_modules[modname]
if astng is None:
raise AstroidBuildingException('Failed to import module %r' % modname)
return astng
def ast_from_module(self, module: types.ModuleType, modname: str = None):
"""given an imported module, return the astroid object"""
modname = modname or module.__name__
if modname in self.astroid_cache:
return self.astroid_cache[modname]
try:
# some builtin modules don't have __file__ attribute
filepath = module.__file__
if is_python_source(filepath):
return self.ast_from_file(filepath, modname)
except AttributeError:
pass
# pylint: disable=import-outside-toplevel; circular import
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).module_build(module, modname)
def _add_transform(package_name, *class_names):
"""Transform package's classes."""
transforms_dir = os.path.join(os.path.dirname(__file__), 'transforms')
fake_module_path = os.path.join(transforms_dir, '%s.py' % re.sub(r'\.', '_', package_name))
with open(fake_module_path) as modulefile:
fake_module = modulefile.read()
fake = AstroidBuilder(MANAGER).string_build(fake_module)
def set_fake_locals(module):
"""Set fake locals for package."""
if module.name != package_name:
return
for class_name in class_names:
module.locals[class_name] = fake.locals[class_name]
MANAGER.register_transform(nodes.Module, set_fake_locals)
def hashlib_transform(module):
template = '''
class %s(object):
def __init__(self, value=''): pass
def digest(self):
return u''
def update(self, value): pass
def hexdigest(self):
return u''
'''
algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
classes = "".join(template % hashfunc for hashfunc in algorithms)
fake = AstroidBuilder(MANAGER).string_build(classes)
for hashfunc in algorithms:
module.locals[hashfunc] = fake.locals[hashfunc]
def test_ancestors_patching_class_recursion(self):
node = AstroidBuilder().string_build(textwrap.dedent("""
import string
Template = string.Template
class A(Template):
pass
class B(A):
pass
def test(x=False):
if x:
string.Template = A
else:
string.Template = B
"""))
klass = node['A']
ancestors = list(klass.ancestors())
self.assertEqual(ancestors[0].qname(), 'string.Template')
def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
"""given a module name, return the astroid object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
elif fallback and modname:
return self.ast_from_module_name(modname)
raise AstroidBuildingException('unable to get astroid for file %s' %
filepath)
def infer_named_tuple(node, context=None):
"""Specific inference function for namedtuple Call node"""
class_node, name, attributes = infer_func_form(node, nodes.Tuple._proxied,
context=context)
fake = AstroidBuilder(MANAGER).string_build('''
class %(name)s(tuple):
_fields = %(fields)r
def _asdict(self):
return self.__dict__
@classmethod
def _make(cls, iterable, new=tuple.__new__, len=len):
return new(cls, iterable)
def _replace(self, **kwds):
return self
''' % {'name': name, 'fields': attributes})
class_node._locals['_asdict'] = fake.body[0]._locals['_asdict']
class_node._locals['_make'] = fake.body[0]._locals['_make']
class_node._locals['_replace'] = fake.body[0]._locals['_replace']
class_node._locals['_fields'] = fake.body[0]._locals['_fields']
# we use UseInferenceDefault, we can't be a generator so return an iterator
return iter([class_node])
def _six_fail_hook(modname):
if modname != 'six.moves':
raise AstroidBuildingException
module = AstroidBuilder(MANAGER).string_build(_IMPORTS)
module.name = 'six.moves'
return module
def setUp(self):
self.manager = AstroidManager()
self.builder = AstroidBuilder(self.manager)
self.manager.astroid_cache.clear()
def test_living_property(self):
builder = AstroidBuilder()
builder._done = {}
builder._module = sys.modules[__name__]
builder.object_build(build_module('module_name', ''), Whatever)
def setUp(self):
self.manager = AstroidManager()
self.manager.clear_cache() # take care of borg
self.builder = AstroidBuilder(self.manager)
class Python3TC(TestCase):
def setUp(self):
self.manager = AstroidManager()
self.builder = AstroidBuilder(self.manager)
self.manager.astroid_cache.clear()
@require_version('3.0')
def test_starred_notation(self):
astroid = self.builder.string_build("*a, b = [1, 2, 3]", 'test', 'test')
# Get the star node
node = next(next(next(astroid.get_children()).get_children()).get_children())
self.assertTrue(isinstance(node.ass_type(), Assign))
@require_version('3.3')
def test_yield_from(self):
body = dedent("""
def func():
yield from iter([1, 2])
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertIsInstance(func, Function)
yieldfrom_stmt = func.body[0]
self.assertIsInstance(yieldfrom_stmt, Discard)
self.assertIsInstance(yieldfrom_stmt.value, YieldFrom)
self.assertEqual(yieldfrom_stmt.as_string(),
'yield from iter([1, 2])')
@require_version('3.3')
def test_yield_from_is_generator(self):
body = dedent("""
def func():
yield from iter([1, 2])
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertIsInstance(func, Function)
self.assertTrue(func.is_generator())
@require_version('3.3')
def test_yield_from_as_string(self):
body = dedent("""
def func():
yield from iter([1, 2])
value = yield from other()
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertEqual(func.as_string().strip(), body.strip())
# metaclass tests
@require_version('3.0')
def test_simple_metaclass(self):
astroid = self.builder.string_build("class Test(metaclass=type): pass")
klass = astroid.body[0]
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'type')
@require_version('3.0')
def test_metaclass_error(self):
astroid = self.builder.string_build("class Test(metaclass=typ): pass")
klass = astroid.body[0]
self.assertFalse(klass.metaclass())
@require_version('3.0')
def test_metaclass_imported(self):
astroid = self.builder.string_build(dedent("""
from abc import ABCMeta
class Test(metaclass=ABCMeta): pass"""))
klass = astroid.body[1]
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'ABCMeta')
@require_version('3.0')
def test_as_string(self):
body = dedent("""
from abc import ABCMeta
class Test(metaclass=ABCMeta): pass""")
astroid = self.builder.string_build(body)
klass = astroid.body[1]
self.assertEqual(klass.as_string(),
'\n\nclass Test(metaclass=ABCMeta):\n pass\n')
@require_version('3.0')
def test_old_syntax_works(self):
astroid = self.builder.string_build(dedent("""
class Test:
__metaclass__ = type
class SubTest(Test): pass
"""))
klass = astroid['SubTest']
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'type')
@require_version('3.0')
def test_metaclass_yes_leak(self):
astroid = self.builder.string_build(dedent("""
# notice `ab` instead of `abc`
from ab import ABCMeta
class Meta(metaclass=ABCMeta): pass
"""))
klass = astroid['Meta']
self.assertIsNone(klass.metaclass())
@require_version('3.0')
def test_metaclass_ancestors(self):
astroid = self.builder.string_build(dedent("""
from abc import ABCMeta
class FirstMeta(metaclass=ABCMeta): pass
class SecondMeta(metaclass=type):
pass
class Simple:
pass
class FirstImpl(FirstMeta): pass
class SecondImpl(FirstImpl): pass
class ThirdImpl(Simple, SecondMeta):
pass
"""))
classes = {
'ABCMeta': ('FirstImpl', 'SecondImpl'),
'type': ('ThirdImpl', )
}
for metaclass, names in classes.items():
for name in names:
impl = astroid[name]
meta = impl.metaclass()
self.assertIsInstance(meta, Class)
self.assertEqual(meta.name, metaclass)
class Python3TC(TestCase):
def setUp(self):
self.manager = AstroidManager()
self.builder = AstroidBuilder(self.manager)
self.manager.astroid_cache.clear()
@require_version('3.0')
def test_starred_notation(self):
astroid = self.builder.string_build("*a, b = [1, 2, 3]", 'test', 'test')
# Get the star node
node = next(next(next(astroid.get_children()).get_children()).get_children())
self.assertTrue(isinstance(node.ass_type(), Assign))
@require_version('3.3')
def test_yield_from(self):
body = dedent("""
def func():
yield from iter([1, 2])
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertIsInstance(func, Function)
yieldfrom_stmt = func.body[0]
self.assertIsInstance(yieldfrom_stmt, Discard)
self.assertIsInstance(yieldfrom_stmt.value, YieldFrom)
self.assertEqual(yieldfrom_stmt.as_string(),
'yield from iter([1, 2])')
@require_version('3.3')
def test_yield_from_is_generator(self):
body = dedent("""
def func():
yield from iter([1, 2])
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertIsInstance(func, Function)
self.assertTrue(func.is_generator())
@require_version('3.3')
def test_yield_from_as_string(self):
body = dedent("""
def func():
yield from iter([1, 2])
value = yield from other()
""")
astroid = self.builder.string_build(body)
func = astroid.body[0]
self.assertEqual(func.as_string().strip(), body.strip())
# metaclass tests
@require_version('3.0')
def test_simple_metaclass(self):
astroid = self.builder.string_build("class Test(metaclass=type): pass")
klass = astroid.body[0]
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'type')
@require_version('3.0')
def test_metaclass_error(self):
astroid = self.builder.string_build("class Test(metaclass=typ): pass")
klass = astroid.body[0]
self.assertFalse(klass.metaclass())
@require_version('3.0')
def test_metaclass_imported(self):
astroid = self.builder.string_build(dedent("""
from abc import ABCMeta
class Test(metaclass=ABCMeta): pass"""))
klass = astroid.body[1]
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'ABCMeta')
@require_version('3.0')
def test_as_string(self):
body = dedent("""
from abc import ABCMeta
class Test(metaclass=ABCMeta): pass""")
astroid = self.builder.string_build(body)
klass = astroid.body[1]
self.assertEqual(klass.as_string(),
'\n\nclass Test(metaclass=ABCMeta):\n pass\n')
@require_version('3.0')
def test_old_syntax_works(self):
astroid = self.builder.string_build(dedent("""
class Test:
__metaclass__ = type
class SubTest(Test): pass
"""))
klass = astroid['SubTest']
metaclass = klass.metaclass()
self.assertIsInstance(metaclass, Class)
self.assertEqual(metaclass.name, 'type')
