def fail_hook_erp5_component(modname):
if not modname.startswith('erp5.'):
raise AstroidBuildingException()
if (modname
in ('erp5.portal_type', 'erp5.component', 'erp5.component.module',
'erp5.component.extension', 'erp5.component.document',
'erp5.component.tool', 'erp5.component.interface',
'erp5.component.mixin', 'erp5.component.test')
or (modname.startswith('erp5.component.')
and modname.endswith('_version'))):
module = AstroidBuilder(MANAGER).string_build('', modname)
if modname.startswith('erp5.component'):
module.package = True
else:
module = _buildAstroidModuleFromComponentModuleName(modname)
return module
def test_numpy_crash(self):
"""test don't crash on numpy"""
# a crash occurred somewhere in the past, and an
# InferenceError instead of a crash was better, but now we even infer!
try:
import numpy # pylint: disable=unused-import
except ImportError:
self.skipTest("test skipped: numpy is not available")
builder = AstroidBuilder()
data = """
from numpy import multiply
multiply(1, 2, 3)
"""
astroid = builder.string_build(data, __name__, __file__)
callfunc = astroid.body[1].value.func
inferred = callfunc.inferred()
self.assertEqual(len(inferred), 1)
def infer_enum_class(node):
""" Specific inference for enums. """
names = set(('Enum', 'IntEnum', 'enum.Enum', 'enum.IntEnum'))
for basename in node.basenames:
# TODO: doesn't handle subclasses yet. This implementation
# is a hack to support enums.
if basename not in names:
continue
if node.root().name == 'enum':
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssignName)
for value in values):
continue
stmt = values[0].statement()
if isinstance(stmt.targets[0], nodes.Tuple):
targets = stmt.targets[0].itered()
else:
targets = stmt.targets
new_targets = []
for target in targets:
# Replace all the assignments with our mocked class.
classdef = dedent('''
class %(name)s(%(types)s):
@property
def value(self):
# Not the best return.
return None
@property
def name(self):
return %(name)r
''' % {'name': target.name, 'types': ', '.join(node.basenames)})
fake = AstroidBuilder(MANAGER).string_build(classdef)[target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instantiate_class())
node.locals[local] = new_targets
break
return node
def _buildAstroidModuleFromComponentModuleName(modname):
from Products.ERP5.ERP5Site import getSite
from Acquisition import aq_base
portal = getSite()
component_tool = aq_base(portal.portal_components)
component_obj = None
component_id = modname[len('erp5.component.'):]
if '_version' in modname:
try:
obj = getattr(component_tool,
component_id.replace('_version', '', 1))
except AttributeError:
raise AstroidBuildingException
if obj.getValidationState() in ('modified', 'validated'):
component_obj = obj
else:
raise AstroidBuildingException
else:
try:
package, reference = component_id.split('.', 1)
except ValueError:
raise AstroidBuildingException
for version in portal.getVersionPriorityNameList():
try:
obj = getattr(component_tool,
'%s.%s.%s' % (package, version, reference))
except AttributeError:
continue
if obj.getValidationState() in ('modified', 'validated'):
component_obj = obj
break
if component_obj is None:
raise AstroidBuildingException
# module_build() could also be used but this requires importing
# the ZODB Component and also monkey-patch it to support PEP-302
# for __file__ starting with '<'
module = AstroidBuilder(MANAGER).string_build(
component_obj.getTextContent(validated_only=True), modname)
return module
def test_new_style_class_detection(self):
try:
import pygtk
except ImportError:
self.skipTest('test skipped: pygtk is not available')
# XXX may fail on some pygtk version, because objects in
# gobject._gobject have __module__ set to gobject :(
builder = AstroidBuilder()
data = """
import pygtk
pygtk.require("2.6")
import gobject
class A(gobject.GObject):
pass
"""
astroid = builder.string_build(data, __name__, __file__)
a = astroid['A']
self.assertTrue(a.newstyle)
def ast_from_module(self,
module: types.ModuleType,
modname: Optional[str] = None):
"""given an imported module, return the astroid object"""
modname = modname or module.__name__
if modname in self.astroid_cache:
return self.astroid_cache[modname]
try:
# some builtin modules don't have __file__ attribute
filepath = module.__file__
if is_python_source(filepath):
return self.ast_from_file(filepath, modname)
except AttributeError:
pass
# pylint: disable=import-outside-toplevel; circular import
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).module_build(module, modname)
def __init__(self, type_constraints) -> None:
"""Initialize a type store with all the built-in types from the typeshed module."""
self.type_constraints = type_constraints
self.classes = defaultdict(lambda: defaultdict(list))
self.functions = defaultdict(list)
self.methods = defaultdict(list)
builder = AstroidBuilder()
module = builder.file_build(TYPE_SHED_PATH)
self._parse_classes(module)
self._parse_functions(module)
# Add in initializers
for klass_name, methods in self.classes.items():
if '__init__' in methods:
self.functions[klass_name] = [
class_callable(init) for init, _ in methods['__init__']
]
def _add_transform(package_name, *class_names):
"""Transform package's classes."""
transforms_dir = os.path.join(os.path.dirname(__file__), 'transforms')
fake_module_path = os.path.join(transforms_dir,
'%s.py' % re.sub(r'\.', '_', package_name))
with open(fake_module_path) as modulefile:
fake_module = modulefile.read()
fake = AstroidBuilder(MANAGER).string_build(fake_module)
def set_fake_locals(module):
"""Set fake locals for package."""
if module.name != package_name:
return
for class_name in class_names:
module._locals[class_name] = fake._locals[class_name] # pylint: disable=protected-access
MANAGER.register_transform(nodes.Module, set_fake_locals)
def hashlib_transform(module):
template = '''
class %s(object):
def __init__(self, value=''): pass
def digest(self):
return u''
def update(self, value): pass
def hexdigest(self):
return u''
'''
algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
classes = "".join(template % hashfunc for hashfunc in algorithms)
fake = AstroidBuilder(MANAGER).string_build(classes)
for hashfunc in algorithms:
module.locals[hashfunc] = fake.locals[hashfunc]
def test_ancestors_patching_class_recursion(self):
node = AstroidBuilder().string_build(textwrap.dedent("""
import string
Template = string.Template
class A(Template):
pass
class B(A):
pass
def test(x=False):
if x:
string.Template = A
else:
string.Template = B
"""))
klass = node['A']
ancestors = list(klass.ancestors())
self.assertEqual(ancestors[0].qname(), 'string.Template')
def test_recursion_regression_issue25(self):
builder = AstroidBuilder()
data = """
import recursion as base
_real_Base = base.Base
class Derived(_real_Base):
pass
def run():
base.Base = Derived
"""
astroid = builder.string_build(data, __name__, __file__)
# Used to crash in _is_metaclass, due to wrong
# ancestors chain
classes = astroid.nodes_of_class(nodes.Class)
for klass in classes:
# triggers the _is_metaclass call
klass.type
def infer_named_tuple(node, context=None):
"""Specific inference function for namedtuple Call node"""
class_node, name, attributes = infer_func_form(node, nodes.Tuple._proxied,
context=context)
call_site = arguments.CallSite.from_call(node)
func = next(extract_node('import collections; collections.namedtuple').infer())
try:
rename = next(call_site.infer_argument(func, 'rename', context)).bool_value()
except InferenceError:
rename = False
if rename:
attributes = _get_renamed_namedtuple_atributes(attributes)
field_def = (" {name} = property(lambda self: self[{index:d}], "
"doc='Alias for field number {index:d}')")
field_defs = '\n'.join(field_def.format(name=name, index=index)
for index, name in enumerate(attributes))
fake = AstroidBuilder(MANAGER).string_build('''
class %(name)s(tuple):
__slots__ = ()
_fields = %(fields)r
def _asdict(self):
return self.__dict__
@classmethod
def _make(cls, iterable, new=tuple.__new__, len=len):
return new(cls, iterable)
def _replace(self, **kwds):
return self
def __getnewargs__(self):
return tuple(self)
%(field_defs)s
''' % {'name': name, 'fields': attributes, 'field_defs': field_defs})
class_node.locals['_asdict'] = fake.body[0].locals['_asdict']
class_node.locals['_make'] = fake.body[0].locals['_make']
class_node.locals['_replace'] = fake.body[0].locals['_replace']
class_node.locals['_fields'] = fake.body[0].locals['_fields']
for attr in attributes:
class_node.locals[attr] = fake.body[0].locals[attr]
# we use UseInferenceDefault, we can't be a generator so return an iterator
return iter([class_node])
def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
"""given a module name, return the astroid object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
elif fallback and modname:
return self.ast_from_module_name(modname)
raise AstroidBuildingException('unable to get astroid for file %s' %
filepath)
def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
"""given a module name, return the astroid object"""
try:
filepath = modutils.get_source_file(filepath, include_no_ext=True)
source = True
except modutils.NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modutils.modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
return self.astroid_cache[modname]
if source:
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).file_build(filepath, modname)
if fallback and modname:
return self.ast_from_module_name(modname)
raise exceptions.AstroidBuildingError(
'Unable to build an AST for {path}.', path=filepath)
def infer_named_tuple(node, context=None):
"""Specific inference function for namedtuple Call node"""
class_node, name, attributes = infer_func_form(node, nodes.Tuple._proxied,
context=context)
fake = AstroidBuilder(MANAGER).string_build('''
class %(name)s(tuple):
_fields = %(fields)r
def _asdict(self):
return self.__dict__
@classmethod
def _make(cls, iterable, new=tuple.__new__, len=len):
return new(cls, iterable)
def _replace(self, **kwds):
return self
''' % {'name': name, 'fields': attributes})
class_node._locals['_asdict'] = fake.body[0]._locals['_asdict']
class_node._locals['_make'] = fake.body[0]._locals['_make']
class_node._locals['_replace'] = fake.body[0]._locals['_replace']
class_node._locals['_fields'] = fake.body[0]._locals['_fields']
# we use UseInferenceDefault, we can't be a generator so return an iterator
return iter([class_node])
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in ('.zip', '.egg'):
try:
eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace(os.path.sep, '.')
if importer.is_package(resource):
zmodname = zmodname + '.__init__'
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except:
continue
return None
def infer_enum_class(node, context=None):
""" Specific inference for enums. """
names = set(('Enum', 'IntEnum', 'enum.Enum', 'enum.IntEnum'))
for basename in node.basenames:
# TODO: doesn't handle subclasses yet.
if basename not in names:
continue
if node.root().name == 'enum':
# Skip if the class is directly from enum module.
break
for local, values in node.locals.items():
if any(not isinstance(value, nodes.AssName) for value in values):
continue
parent = values[0].parent
real_value = parent.value
new_targets = []
for target in parent.targets:
# Replace all the assignments with our mocked class.
classdef = dedent('''
class %(name)s(object):
@property
def value(self):
return %(value)s
@property
def name(self):
return %(name)r
%(name)s = %(value)s
''' % {
'name': target.name,
'value': real_value.as_string()
})
fake = AstroidBuilder(MANAGER).string_build(classdef)[
target.name]
fake.parent = target.parent
for method in node.mymethods():
fake.locals[method.name] = [method]
new_targets.append(fake.instanciate_class())
node.locals[local] = new_targets
break
return node
def pkg_resources_transform(module):
fake = AstroidBuilder(MANAGER).string_build('''
def resource_exists(package_or_requirement, resource_name):
pass
def resource_isdir(package_or_requirement, resource_name):
pass
def resource_filename(package_or_requirement, resource_name):
pass
def resource_stream(package_or_requirement, resource_name):
pass
def resource_string(package_or_requirement, resource_name):
pass
def resource_listdir(package_or_requirement, resource_name):
pass
def extraction_error():
pass
def get_cache_path(archive_name, names=()):
pass
def postprocess(tempname, filename):
pass
def set_extraction_path(path):
pass
def cleanup_resources(force=False):
pass
''')
for func_name, func in fake.locals.items():
module.locals[func_name] = func
def zip_import_data(self, filepath):
if zipimport is None:
return None
from astroid.builder import AstroidBuilder
builder = AstroidBuilder(self)
for ext in (".zip", ".egg"):
try:
eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace(os.path.sep, ".")
if importer.is_package(resource):
zmodname = zmodname + ".__init__"
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except Exception: # pylint: disable=broad-except
continue
return None
def hashlib_transform():
template = '''
class %(name)s(object):
def __init__(self, value=''): pass
def digest(self):
return %(digest)s
def copy(self):
return self
def update(self, value): pass
def hexdigest(self):
return ''
@property
def name(self):
return %(name)r
'''
algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
classes = "".join(template % {
'name': hashfunc,
'digest': 'b""' if PY3K else '""'
} for hashfunc in algorithms)
return AstroidBuilder(MANAGER).string_build(classes)
def from_setup_py(setup_file):
try:
from astroid import AstroidBuildingException
except ImportError:
syntax_exceptions = (SyntaxError, )
else:
syntax_exceptions = (SyntaxError, AstroidBuildingException)
try:
contents = _load_file_contents(setup_file)
ast = AstroidBuilder(MANAGER).string_build(contents)
except syntax_exceptions:
# if the setup file is broken, we can't do much about that...
raise CouldNotParseRequirements
walker = SetupWalker(ast)
requirements = []
for req in walker.get_requires():
requirements.append(DetectedRequirement.parse(req, setup_file))
return requirements
def pytest_transform():
return AstroidBuilder(MANAGER).string_build('''
try:
import _pytest.mark
import _pytest.recwarn
import _pytest.runner
import _pytest.python
except ImportError:
pass
else:
deprecated_call = _pytest.recwarn.deprecated_call
exit = _pytest.runner.exit
fail = _pytest.runner.fail
fixture = _pytest.python.fixture
importorskip = _pytest.runner.importorskip
mark = _pytest.mark.MarkGenerator()
raises = _pytest.python.raises
skip = _pytest.runner.skip
yield_fixture = _pytest.python.yield_fixture
''')
def _add_transform(package_name, *class_names):
"""Transform package's classes."""
transforms_dir = os.path.join(os.path.dirname(__file__), 'transforms')
fake_module_path = os.path.join(transforms_dir,
'%s.py' % re.sub(r'\.', '_', package_name))
with open(fake_module_path) as modulefile:
fake_module = modulefile.read()
fake = AstroidBuilder(MANAGER).string_build(fake_module)
def set_fake_locals(module):
"""Set fake locals for package."""
if module.name != package_name:
return
for class_name in class_names:
# This changed from locals to _locals between astroid 1.3 and 1.4
if hasattr(module, '_locals'):
module._locals[class_name].extend(fake._locals[class_name]) # pylint: disable=protected-access
else:
module.locals[class_name].extend(fake.locals[class_name])
MANAGER.register_transform(nodes.Module, set_fake_locals)
def _import_gi_module(modname):
# we only consider gi.repository submodules
if not modname.startswith('gi.repository.'):
raise AstroidBuildingException()
# build astroid representation unless we already tried so
if modname not in _inspected_modules:
modnames = [modname]
optional_modnames = []
# GLib and GObject may have some special case handling
# in pygobject that we need to cope with. However at
# least as of pygobject3-3.13.91 the _glib module doesn't
# exist anymore, so if treat these modules as optional.
if modname == 'gi.repository.GLib':
optional_modnames.append('gi._glib')
elif modname == 'gi.repository.GObject':
optional_modnames.append('gi._gobject')
try:
modcode = ''
for m in itertools.chain(modnames, optional_modnames):
try:
__import__(m)
modcode += _gi_build_stub(sys.modules[m])
except ImportError:
if m not in optional_modnames:
raise
except ImportError:
astng = _inspected_modules[modname] = None
else:
astng = AstroidBuilder(MANAGER).string_build(modcode, modname)
_inspected_modules[modname] = astng
else:
astng = _inspected_modules[modname]
if astng is None:
raise AstroidBuildingException('Failed to import module %r' % modname)
return astng
def _new_import_module(self, modname, relative_only=False, level=None):
# Could be a static piece of gi.repository or whatever unrelated module,
# let that fall through
try:
return _orig_import_module(self, modname, relative_only, level)
except AstroidBuildingException:
# we only consider gi.repository submodules
if not modname.startswith('gi.repository.'):
if relative_only and level is None:
level = 0
modname = self.relative_to_absolute_name(modname, level)
if not modname.startswith('gi.repository.'):
raise
# build astroid representation unless we already tried so
if modname not in _inspected_modules:
modnames = [modname]
# GLib and GObject have some special case handling
# in pygobject that we need to cope with
if modname == 'gi.repository.GLib':
modnames.append('gi._glib')
elif modname == 'gi.repository.GObject':
modnames.append('gi._gobject')
try:
modcode = ''
for m in modnames:
__import__(m)
modcode += _gi_build_stub(sys.modules[m])
except ImportError:
astng = _inspected_modules[modname] = None
else:
astng = AstroidBuilder(MANAGER).string_build(modcode, modname)
_inspected_modules[modname] = astng
else:
astng = _inspected_modules[modname]
if astng is None:
raise AstroidBuildingException('Failed to import module %r' % modname)
return astng
def collections_transform():
return AstroidBuilder(MANAGER).string_build('''
class defaultdict(dict):
default_factory = None
def __missing__(self, key): pass
class deque(object):
maxlen = 0
def __init__(self, iterable=None, maxlen=None): pass
def append(self, x): pass
def appendleft(self, x): pass
def clear(self): pass
def count(self, x): return 0
def extend(self, iterable): pass
def extendleft(self, iterable): pass
def pop(self): pass
def popleft(self): pass
def remove(self, value): pass
def reverse(self): pass
def rotate(self, n): pass
def __iter__(self): return self
''')
def zmq_transform(module):
"""Create a fake zmq module with missing members"""
if module.name == 'zmq':
fake = AstroidBuilder(MANAGER).string_build('''
DEALER = None
ROUTER = None
REQ = None
REP = None
PUB = None
SUB = None
PUSH = None
PULL = None
SUBSCRIBE = None
UNSUBSCRIBE = None
NOBLOCK = None
class ZMQError(Exception):
pass
import zmq.green as zmq
class MySocket(zmq.Socket):
setsockopt_string = lambda x, y: None
connect = lambda x: None
setsockopt = lambda x, y: None
recv_string = lambda: None
send_string = lambda x: None
class Context():
socket = lambda x: MySocket()
''')
for property in ('Context', 'DEALER', 'ROUTER', 'REQ', 'REP', 'PUB',
'SUB', 'PUSH', 'PULL', 'SUBSCRIBE', 'UNSUBSCRIBE',
'NOBLOCK', 'ZMQError'):
module.locals[property] = fake.locals[property]
def ast_from_module_name(self, modname, context_file=None):
"""given a module name, return the astroid object"""
if modname in self.astroid_cache:
return self.astroid_cache[modname]
if modname == '__main__':
from astroid.builder import AstroidBuilder
return AstroidBuilder(self).string_build('', modname)
old_cwd = os.getcwd()
if context_file:
os.chdir(dirname(context_file))
try:
filepath = self.file_from_module_name(modname, context_file)
if filepath is not None and not is_python_source(filepath):
module = self.zip_import_data(filepath)
if module is not None:
return module
if filepath is None or not is_python_source(filepath):
try:
module = load_module_from_name(modname)
except Exception, ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
raise AstroidBuildingException(msg)
return self.ast_from_module(module, modname)
return self.ast_from_file(filepath, modname, fallback=False)
def cubicweb_abstractmethods_transform(classdef):
if class_is_abstract(classdef):
return
def is_abstract(method):
return method.is_abstract(pass_is_abstract=False)
methods = sorted(
unimplemented_abstract_methods(classdef, is_abstract).items(),
key=lambda item: item[0],
)
dummy_method = AstroidBuilder(MANAGER).string_build('''
def dummy_method(self):
""""""
''')
for name, method in methods:
owner = method.parent.frame()
if owner is classdef:
continue
if name not in classdef.locals:
if name in ('cell_call', 'entity_call', 'render_body'):
classdef.set_local(name, dummy_method)
def ipaplatform_tasks_transform():
return AstroidBuilder(MANAGER).string_build(
textwrap.dedent('''
from ipaplatform.base.tasks import tasks
__all__ = ('tasks',)
'''))
