def hashlib_transform(module):
if module.name == 'hashlib':
fake = ASTNGBuilder(MANAGER).string_build('''
class fakehash(object):
digest_size = -1
def __init__(self, value): pass
def digest(self):
return u''
def hexdigest(self):
return u''
def update(self, value): pass
class md5(fakehash):
pass
class sha1(fakehash):
pass
class sha256(fakehash):
pass
''')
for hashfunc in ('sha256', 'sha1', 'md5'):
module.locals[hashfunc] = fake.locals[hashfunc]
def astng_from_file(self, filepath, modname=None, fallback=True):
"""given a module name, return the astng object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
source = False
if modname is None:
modname = '.'.join(modpath_from_file(filepath))
if modname in self._cache:
return self._cache[modname]
if source:
try:
from logilab.astng.builder import ASTNGBuilder
return ASTNGBuilder(self).file_build(filepath, modname)
except (SyntaxError, KeyboardInterrupt, SystemExit):
raise
except Exception, ex:
raise
if __debug__:
print 'error while building astng for', filepath
import traceback
traceback.print_exc()
msg = 'Unable to load module %s (%s)' % (modname, ex)
raise ASTNGBuildingException(msg), None, sys.exc_info()[-1]
def astng_from_file(self,
filepath,
modname=None,
fallback=True,
source=False):
"""given a module name, return the astng object"""
try:
filepath = get_source_file(filepath, include_no_ext=True)
source = True
except NoSourceFile:
pass
if modname is None:
try:
modname = '.'.join(modpath_from_file(filepath))
except ImportError:
modname = filepath
if modname in self.astng_cache:
return self.astng_cache[modname]
if source:
from logilab.astng.builder import ASTNGBuilder
return ASTNGBuilder(self).file_build(filepath, modname)
elif fallback and modname:
return self.astng_from_module_name(modname)
raise ASTNGBuildingException('unable to get astng for file %s' %
filepath)
def subprocess_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
class Popen(object):
returncode = pid = 0
stdin = stdout = stderr = file()
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
pass
def communicate(self, input=None):
return ('string', 'string')
def wait(self):
return self.returncode
def poll(self):
return self.returncode
def send_signal(self, signal):
pass
def terminate(self):
pass
def kill(self):
pass
''')
for func_name, func in fake.locals.items():
module.locals[func_name] = func
def collections_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
class defaultdict(dict):
default_factory = None
def __missing__(self, key): pass
class deque(object):
maxlen = 0
def __init__(iterable=None, maxlen=None): pass
def append(self, x): pass
def appendleft(self, x): pass
def clear(self): pass
def count(self, x): return 0
def extend(self, iterable): pass
def extendleft(self, iterable): pass
def pop(self): pass
def popleft(self): pass
def remove(self, value): pass
def reverse(self): pass
def rotate(self, n): pass
''')
for klass in ('deque', 'defaultdict'):
module.locals[klass] = fake.locals[klass]
def astng_from_module_name(self, modname, context_file=None):
"""given a module name, return the astng object"""
if modname in self.astng_cache:
return self.astng_cache[modname]
if modname == '__main__':
from logilab.astng.builder import ASTNGBuilder
return ASTNGBuilder(self).string_build('', modname)
old_cwd = os.getcwd()
if context_file:
os.chdir(dirname(context_file))
try:
filepath = self.file_from_module_name(modname, context_file)
if filepath is not None and not is_python_source(filepath):
module = self.zip_import_data(filepath)
if module is not None:
return module
if filepath is None or not is_python_source(filepath):
try:
module = load_module_from_name(modname)
except Exception as ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
raise ASTNGBuildingException(msg)
return self.astng_from_module(module, modname)
return self.astng_from_file(filepath, modname, fallback=False)
finally:
os.chdir(old_cwd)
def urlparse_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
def urlparse(url, scheme='', allow_fragments=True):
return ParseResult()
class ParseResult(object):
def __init__(self):
self.scheme = ''
self.netloc = ''
self.path = ''
self.params = ''
self.query = ''
self.fragment = ''
self.username = None
self.password = None
self.hostname = None
self.port = None
def geturl(self):
return ''
''')
for func_name, func in fake.locals.items():
module.locals[func_name] = func
def astng_from_module_name(self, modname, context_file=None):
"""given a module name, return the astng object"""
if modname in self._cache:
return self._cache[modname]
old_cwd = os.getcwd()
if context_file:
os.chdir(dirname(context_file))
try:
filepath = self.file_from_module_name(modname, context_file)
if filepath is not None and not is_python_source(filepath):
data, zmodname = zip_import_data(filepath)
if data is not None:
from logilab.astng.builder import ASTNGBuilder
try:
return ASTNGBuilder(self).string_build(
data, zmodname, filepath)
except (SyntaxError, KeyboardInterrupt, SystemExit):
raise
if filepath is None or not is_python_source(filepath):
try:
module = load_module_from_name(modname)
# catch SystemError as well, we may get that on badly
# initialized C-module
except (SystemError, ImportError), ex:
msg = 'Unable to load module %s (%s)' % (modname, ex)
raise ASTNGBuildingException(msg)
return self.astng_from_module(module, modname)
return self.astng_from_file(filepath, modname, fallback=False)
def pyqt4_qtcore_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
def SIGNAL(signal_name): pass
class QObject(object):
def emit(self, signal): pass
''')
for klass in ('QObject',):
module.locals[klass] = fake.locals[klass]
def sh_transform(module):
if module.name == 'sh':
for name, items in ASTNGBuilder(MANAGER).string_build(
FAKE_SH).locals.items():
for item in items:
item.parent = module
module.locals[name] = items
for item in ('ErrorReturnCode_128', ):
module.locals[item] = [scoped_nodes.Class(item, None)]
def hashlib_transform(module):
if module.name == 'django.utils.translation':
fake = ASTNGBuilder(MANAGER).string_build('''
def ugettext_lazy(value):
return u''
def ugettext(value):
return u''
''')
for hashfunc in ('ugettext_lazy', 'ugettext'):
module.locals[hashfunc] = fake.locals[hashfunc]
def mechanize_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
class Browser(object):
def open(self, url, data=None, timeout=None):
return None
def open_novisit(self, url, data=None, timeout=None):
return None
def open_local_file(self, filename):
return None
''')
module.locals['Browser'] = fake.locals['Browser']
def astng_from_module(self, module, modname=None):
"""given an imported module, return the astng object"""
modname = modname or module.__name__
if modname in self._cache:
return self._cache[modname]
try:
# some builtin modules don't have __file__ attribute
filepath = module.__file__
if is_python_source(filepath):
return self.astng_from_file(filepath, modname)
except AttributeError:
pass
from logilab.astng.builder import ASTNGBuilder
return ASTNGBuilder(self).module_build(module, modname)
def hashlib_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
class md5(object):
def __init__(self, value): pass
def hexdigest(self):
return u''
class sha1(object):
def __init__(self, value): pass
def hexdigest(self):
return u''
''')
for hashfunc in ('sha1', 'md5'):
module.locals[hashfunc] = fake.locals[hashfunc]
def test_numpy_crash(self):
try:
import numpy
except ImportError:
self.skipTest('test skipped: numpy is not available')
else:
builder = ASTNGBuilder()
data = """
from numpy import multiply
multiply(1, 2, 3)
"""
astng = builder.string_build(data, __name__, __file__)
callfunc = astng.node.nodes[1].expr
# well, InferenceError instead of a crash is better
self.assertRaises(InferenceError, list, callfunc.infer())
def test_new_style_class_detection(self):
try:
import pygtk
except ImportError:
self.skipTest('test skipped: pygtk is not available')
# XXX may fail on some pygtk version, because objects in
# gobject._gobject have __module__ set to gobject :(
builder = ASTNGBuilder()
data = """
import pygtk
pygtk.require("2.6")
import gobject
class A(gobject.GObject):
pass
"""
astng = builder.string_build(data, __name__, __file__)
a = astng['A']
self.assertTrue(a.newstyle)
def test_numpy_crash(self):
"""test don't crash on numpy"""
#a crash occured somewhere in the past, and an
# InferenceError instead of a crash was better, but now we even infer!
try:
import numpy
except ImportError:
self.skipTest('test skipped: numpy is not available')
builder = ASTNGBuilder()
data = """
from numpy import multiply
multiply(1, 2, 3)
"""
astng = builder.string_build(data, __name__, __file__)
callfunc = astng.body[1].value.func
infered = callfunc.infered()
self.assertEqual(len(infered), 1)
self.assertIsInstance(infered[0], Instance)
def zip_import_data(self, filepath):
if zipimport is None:
return None
from logilab.astng.builder import ASTNGBuilder
builder = ASTNGBuilder(self)
for ext in ('.zip', '.egg'):
try:
eggpath, resource = filepath.rsplit(ext + '/', 1)
except ValueError:
continue
try:
importer = zipimport.zipimporter(eggpath + ext)
zmodname = resource.replace('/', '.')
if importer.is_package(resource):
zmodname = zmodname + '.__init__'
module = builder.string_build(importer.get_source(resource),
zmodname, filepath)
return module
except:
continue
return None
def pkg_resources_transform(module):
fake = ASTNGBuilder(MANAGER).string_build('''
def resource_exists(package_or_requirement, resource_name):
pass
def resource_isdir(package_or_requirement, resource_name):
pass
def resource_filename(package_or_requirement, resource_name):
pass
def resource_stream(package_or_requirement, resource_name):
pass
def resource_string(package_or_requirement, resource_name):
pass
def resource_listdir(package_or_requirement, resource_name):
pass
def extraction_error():
pass
def get_cache_path(archive_name, names=()):
pass
def postprocess(tempname, filename):
pass
def set_extraction_path(path):
pass
def cleanup_resources(force=False):
pass
''')
for func_name, func in fake.locals.items():
module.locals[func_name] = func
class LeConst(LanguageElement):
"""This language element represent a constant element.
At this point I think this can only be a builtin.
"""
from logilab.astng.builder import ASTNGBuilder
from logilab.common.compat import builtins
BUILTINS = ASTNGBuilder().inspect_build(builtins)
KIND = 'b'
def bounded_accessibles(self):
name = self.astng_element.pytype()
if name.startswith('__builtin__'):
name = name.split('.')[-1]
builtin_astng_element = self.BUILTINS[name]
element = LanguageElement.create(
builtin_astng_element,
name=name,
context_string=self.context_string)
return element.bounded_accessibles()
raise RuntimeError("The const type %s is no builtin." %
self.astng_element)
def test_new_style_class_detection(self):
try:
import pygtk
except ImportError:
self.skipTest('test skipped: pygtk is not available')
else:
# XXX may fail on some pygtk version, because objects in
# gobject._gobject have __module__ set to gobject :(
builder = ASTNGBuilder()
data = """
import pygtk
pygtk.require("2.6")
import gobject
class A(gobject.GObject):
def __init__(self, val):
gobject.GObject.__init__(self)
self._val = val
def _get_val(self):
print "get"
return self._val
def _set_val(self, val):
print "set"
self._val = val
val = property(_get_val, _set_val)
if __name__ == "__main__":
print gobject.GObject.__bases__
a = A(7)
print a.val
a.val = 6
print a.val
"""
astng = builder.string_build(data, __name__, __file__)
a = astng['A']
self.failUnless(a.newstyle)
def build_astng_tree(self):
"""build astng tree from the _ast tree
"""
from logilab.astng.builder import ASTNGBuilder
tree = ASTNGBuilder().string_build(self.sourcecode)
return tree
def hashlib_transform(module):
if module.name == 'hashlib':
fake = ASTNGBuilder(MANAGER).string_build(CODE_FIX)
for hashfunc in ('sha1', 'md5', 'sha512'):
module.locals[hashfunc] = fake.locals[hashfunc]
def scapy_transform(module):
if module.name == 'scapy.all':
fake = ASTNGBuilder(MANAGER).string_build(CODE_FIX)
for func in ('IP', 'TCP', 'UDP', 'traceroute'):
module.locals[func] = fake.locals[func]
def gi_repository_transform(module):
if module.name == 'gi.repository':
fake = ASTNGBuilder(MANAGER).string_build(CODE_FIX)
for func in ('Notify', ):
module.locals[func] = fake.locals[func]
def setUp(self):
self.manager = ASTNGManager()
self.builder = ASTNGBuilder(self.manager)
self.manager.astng_cache.clear()
class PyModule(object):
BUILDER = ASTNGBuilder()
def __init__(self, module):
self.astng_module = module
def scope(self, linenumber, scope=None):
scope = scope or self.astng_module
for a_scope in scope.values():
if a_scope.fromlineno <= linenumber <= a_scope.tolineno:
scope = self.scope(linenumber, a_scope)
break
if isinstance(scope, language_elements.LanguageElement):
return scope
return language_elements.LanguageElement.create(scope)
@classmethod
def by_source(cls, source, linenumber=None):
if linenumber is not None:
source_lines = source.split('\n')
original_line = source_lines[linenumber]
indention = indention_by_line(original_line)
for fill in ('pass', 'except: pass', 'except:', '', original_line):
try:
source_lines[linenumber] = indention + fill
module = cls.BUILDER.string_build('\n'.join(source_lines))
return cls(module)
except:
pass
raise NotImplementedError("TODO: Can't parse file")
else:
raise RuntimeError("No line number given.")
return None
@classmethod
def by_module_path(cls, module_path):
helper_module = cls.BUILDER.string_build('') # TODO: bad
module = helper_module.import_module(module_path)
return cls(module)
def package_modules(self):
result = set()
if self.astng_module.package:
package_dir = os.path.dirname(self.astng_module.file)
for module_file in os.listdir(package_dir):
module_path = os.path.join(package_dir, module_file)
module_name = is_module_or_package(module_path)
if module_name:
result.add(completionable.Completionable(module_name))
return result
def accessible_modules(self):
result = set()
for accessible in self.accessibles():
if accessible.__class__.__name__ in ('LeModule', 'LeImport'):
result.add(accessible)
return result
def accessibles(self):
module = language_elements.LanguageElement.create(self.astng_module)
return module.accessibles()
def __init__(self, module):
self.builder = ASTNGBuilder(MANAGER)
self.module = module
def httpretty_transform(module):
if module.name == 'httpretty':
fake = ASTNGBuilder(MANAGER).string_build(CODE_FIX)
for hashfunc in ('enable', 'disable', 'register_uri', 'GET', 'POST'):
module.locals[hashfunc] = fake.locals[hashfunc]
def test_living_property(self):
builder = ASTNGBuilder()
builder._done = {}
builder._module = sys.modules[__name__]
builder.object_build(build_module('module_name', ''), Whatever)
