def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def brainless_manager(self):
manager = ASTNGManager()
# avoid caching into the ASTNGManager borg since we get problems
# with other tests :
manager.__dict__ = {}
manager.astng_cache = {}
manager._mod_file_cache = {}
return manager
def test_borg(self):
"""test that the ASTNGManager is really a borg, i.e. that two different
instances has same cache"""
first_manager = ASTNGManager()
built = first_manager.astng_from_module_name(BUILTINS_NAME)
second_manager = ASTNGManager()
second_built = first_manager.astng_from_module_name(BUILTINS_NAME)
self.assertTrue(built is second_built)
def test_borg(self):
"""test that the ASTNGManager is really a borg, i.e. that two different
instances has same cache"""
first_manager = ASTNGManager()
built = first_manager.astng_from_module_name(BUILTINS_NAME)
second_manager = ASTNGManager()
second_built = first_manager.astng_from_module_name(BUILTINS_NAME)
self.assertTrue(built is second_built)
class PyreverseCommand(ConfigurationMixIn):
"""base class providing common behaviour for pyreverse commands"""
options = OPTIONS
def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def run(self, args):
"""checking arguments and run project"""
if not args:
print self.help()
return
project = self.manager.project_from_files(args, astng_wrapper)
linker = Linker(project, tag=True)
handler = DiadefsHandler(self.config)
diadefs = handler.get_diadefs(project, linker)
if self.config.output_format == "vcg":
writer.VCGWriter(self.config).write(diadefs)
else:
writer.DotWriter(self.config).write(diadefs)
class PyreverseCommand(ConfigurationMixIn):
"""base class providing common behaviour for pyreverse commands"""
options = OPTIONS
def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def run(self, args):
"""checking argmuents and run project"""
if not args:
print self.help()
return
project = self.manager.project_from_files(args, astng_wrapper)
linker = Linker(project, tag=True)
handler = DiadefsHandler(self.config)
diadefs = handler.get_diadefs(project, linker)
if self.config.output_format == "vcg":
writer.VCGWriter(self.config).write(diadefs)
else:
writer.DotWriter(self.config).write(diadefs)
def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
class PyreverseCommand(ConfigurationMixIn):
"""base class providing common behaviour for pyreverse commands"""
options = OPTIONS
def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def run(self, args):
"""checking arguments and run project"""
if not args:
print(self.help())
return
# insert current working directory to the python path to recognize
# dependencies to local modules even if cwd is not in the PYTHONPATH
sys.path.insert(0, os.getcwd())
try:
project = self.manager.project_from_files(args)
linker = Linker(project, tag=True)
handler = DiadefsHandler(self.config)
diadefs = handler.get_diadefs(project, linker)
finally:
sys.path.pop(0)
if self.config.output_format == "vcg":
writer.VCGWriter(self.config).write(diadefs)
else:
writer.DotWriter(self.config).write(diadefs)
def __init__(self, args):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
project = self.manager.project_from_files(args, astng_wrapper)
self.project = project
def __init__(self, view, args, callback):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
self.view = view
self.callback = callback
self.run(args)
def __init__(self, args,process_candidates=False):
ConfigurationMixIn.__init__(self, usage=__doc__)
self._prob_used_classes = Set([])
self._dbg_assattr_parents = Set([])
self._process_candidates = process_candidates
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def __init__(self, args,criteria='default',out_file='test.xml',treshold=None,add_value=False):
ConfigurationMixIn.__init__(self, usage=__doc__)
self._add_value = add_value
self._project = args[0]
self._treshold = treshold
self._out_file = out_file
self._criteria = criteria
self._dbg_assattr_parents = set([])
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
def brainless_manager(self):
manager = ASTNGManager()
# avoid caching into the ASTNGManager borg since we get problems
# with other tests :
manager.__dict__ = {}
manager.astng_cache = {}
manager._mod_file_cache = {}
manager.transformers = {}
return manager
class ScannerCommand(ConfigurationMixIn):
"""base class providing common behaviour for pyreverse commands"""
options = OPTIONS
def __init__(self, view, args, callback):
ConfigurationMixIn.__init__(self, usage=__doc__)
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
self.view = view
self.callback = callback
self.run(args)
def run(self, args):
"""checking arguments and run project"""
if not args:
print self.help()
return
# insert current working directory to the python path to recognize
# dependencies to local modules even if cwd is not in the PYTHONPATH
sys.path.insert(0, args[0])
sys.path.insert(0, os.getcwd())
try:
project = self.manager.project_from_files(args, black_list= \
map(os.path.relpath, BlackList.blacklist))
linker = Linker(project, tag=True)
handler = DiadefsHandler(self.config)
diadefs = handler.get_diadefs(project, linker)
finally:
sys.path.pop(0)
# filter just classes (not packages) for now
diadefs = filter(lambda x: x.TYPE == 'class', diadefs)
# update GUI
Gdk.threads_enter()
self.callback()
writer.CanvasWriter(self.view, self.config).write(diadefs)
Gdk.threads_leave()
class ASTNGManagerTC(TestCase):
def setUp(self):
self.manager = ASTNGManager()
self.manager.astng_cache.clear()
def test_astng_from_module(self):
import unittest
astng = self.manager.astng_from_module(unittest)
self.assertEqual(astng.pure_python, True)
import time
astng = self.manager.astng_from_module(time)
self.assertEqual(astng.pure_python, False)
def test_astng_from_class(self):
astng = self.manager.astng_from_class(int)
self.assertEqual(astng.name, "int")
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
astng = self.manager.astng_from_class(object)
self.assertEqual(astng.name, "object")
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
self.failUnless("__setattr__" in astng)
def _test_astng_from_zip(self, archive):
origpath = sys.path[:]
sys.modules.pop("mypypa", None)
archive_path = join(DATA, archive)
sys.path.insert(0, archive_path)
try:
module = self.manager.astng_from_module_name("mypypa")
self.assertEqual(module.name, "mypypa")
self.failUnless(module.file.endswith("%s/mypypa" % archive), module.file)
finally:
# remove the module, else after importing egg, we don't get the zip
if "mypypa" in self.manager.astng_cache:
del self.manager.astng_cache["mypypa"]
del self.manager._mod_file_cache[("mypypa", None)]
if archive_path in sys.path_importer_cache:
del sys.path_importer_cache[archive_path]
sys.path = origpath
def test_astng_from_module_name_egg(self):
self._test_astng_from_zip("MyPyPa-0.1.0-py2.5.egg")
def test_astng_from_module_name_zip(self):
self._test_astng_from_zip("MyPyPa-0.1.0-py2.5.zip")
def test_from_directory(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, "data")
self.assertEqual(obj.name, "data")
self.assertEqual(obj.path, join(DATA, "__init__.py"))
def test_project_node(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, "data")
expected = set(
["SSL1", "__init__", "all", "appl", "format", "module", "module2", "noendingnewline", "nonregr", "notall"]
)
expected = [
"data",
"data.SSL1",
"data.SSL1.Connection1",
"data.all",
"data.appl",
"data.appl.myConnection",
"data.format",
"data.module",
"data.module2",
"data.noendingnewline",
"data.nonregr",
"data.notall",
]
self.assertListEqual(sorted(k for k in obj.keys()), expected)
def test_do_not_expose_main(self):
obj = self.manager.astng_from_module_name("__main__")
self.assertEqual(obj.name, "__main__")
self.assertEqual(obj.items(), [])
class ASTNGManagerTC(unittest.TestCase):
def setUp(self):
self.manager = ASTNGManager(borg=False)
def test_astng_from_module(self):
astng = self.manager.astng_from_module(unittest)
self.assertEqual(astng.pure_python, True)
import time
astng = self.manager.astng_from_module(time)
self.assertEqual(astng.pure_python, False)
def test_astng_from_class(self):
astng = self.manager.astng_from_class(file)
self.assertEqual(astng.name, 'file')
self.assertEqual(astng.parent.frame().name, '__builtin__')
astng = self.manager.astng_from_class(object)
self.assertEqual(astng.name, 'object')
self.assertEqual(astng.parent.frame().name, '__builtin__')
self.failUnless('__setattr__' in astng)
def _test_astng_from_zip(self, archive):
origpath = sys.path[:]
sys.modules.pop('mypypa', None)
sys.path.insert(0, join(dirname(__file__), 'data', archive))
try:
module = self.manager.astng_from_module_name('mypypa')
self.assertEqual(module.name, 'mypypa')
self.failUnless(module.file.endswith('%s/mypypa' % archive),
module.file)
finally:
sys.path = origpath
def test_astng_from_module_name_egg(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.egg')
def test_astng_from_module_name_zip(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.zip')
def test_from_directory(self):
obj = self.manager.from_directory('data')
self.assertEqual(obj.name, 'data')
self.assertEqual(obj.path, join(os.getcwd(), 'data'))
def test_package_node(self):
obj = self.manager.from_directory('data')
expected_short = ['SSL1', '__init__', 'all', 'appl', 'format', 'module', 'module2',
'noendingnewline', 'nonregr', 'notall']
expected_long = ['SSL1', 'data', 'data.all', 'appl', 'data.format', 'data.module',
'data.module2', 'data.noendingnewline', 'data.nonregr',
'data.notall']
self.assertEqual(obj.keys(), expected_short)
self.assertEqual([m.name for m in obj.values()], expected_long)
self.assertEqual([m for m in list(obj)], expected_short)
self.assertEqual([(name, m.name) for name, m in obj.items()],
zip(expected_short, expected_long))
self.assertEqual([(name, m.name) for name, m in obj.items()],
zip(expected_short, expected_long))
self.assertEqual('module' in obj, True)
self.assertEqual(obj.has_key('module'), True)
self.assertEqual(obj.get('module').name, 'data.module')
self.assertEqual(obj['module'].name, 'data.module')
self.assertEqual(obj.get('whatever'), None)
self.assertEqual(obj.fullname(), 'data')
def setUp(self):
self.manager = ASTNGManager(borg=False)
def setUp(self):
self.manager = ASTNGManager(borg=False)
class ASTNGManagerTC(TestCase):
def setUp(self):
self.manager = ASTNGManager()
self.manager.astng_cache.clear()
def test_astng_from_module(self):
import unittest
astng = self.manager.astng_from_module(unittest)
self.assertEqual(astng.pure_python, True)
import time
astng = self.manager.astng_from_module(time)
self.assertEqual(astng.pure_python, False)
def test_astng_from_class(self):
astng = self.manager.astng_from_class(int)
self.assertEqual(astng.name, 'int')
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
astng = self.manager.astng_from_class(object)
self.assertEqual(astng.name, 'object')
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
self.failUnless('__setattr__' in astng)
def _test_astng_from_zip(self, archive):
origpath = sys.path[:]
sys.modules.pop('mypypa', None)
archive_path = join(DATA, archive)
sys.path.insert(0, archive_path)
try:
module = self.manager.astng_from_module_name('mypypa')
self.assertEqual(module.name, 'mypypa')
self.failUnless(module.file.endswith('%s/mypypa' % archive),
module.file)
finally:
# remove the module, else after importing egg, we don't get the zip
if 'mypypa' in self.manager.astng_cache:
del self.manager.astng_cache['mypypa']
del self.manager._mod_file_cache[('mypypa', None)]
if archive_path in sys.path_importer_cache:
del sys.path_importer_cache[archive_path]
sys.path = origpath
def test_astng_from_module_name_egg(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.egg')
def test_astng_from_module_name_zip(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.zip')
def test_from_directory(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, 'data')
self.assertEqual(obj.name, 'data')
self.assertEqual(obj.path, join(DATA, '__init__.py'))
def test_project_node(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, 'data')
expected = set(['SSL1', '__init__', 'all', 'appl', 'format', 'module',
'module2', 'noendingnewline', 'nonregr', 'notall'])
expected = ['data', 'data.SSL1', 'data.SSL1.Connection1', 'data.all',
'data.appl', 'data.appl.myConnection', 'data.format',
'data.module', 'data.module2', 'data.noendingnewline',
'data.nonregr', 'data.notall']
self.assertListEqual(sorted(k for k in list(obj.keys())), expected)
def test_do_not_expose_main(self):
obj = self.manager.astng_from_module_name('__main__')
self.assertEqual(obj.name, '__main__')
self.assertEqual(list(obj.items()), [])
"""
__docformat__ = "restructuredtext en"
import sys
from os.path import abspath
from inspect import (getargspec, isdatadescriptor, isfunction, ismethod,
ismethoddescriptor, isclass, isbuiltin)
from logilab.astng import BUILTINS_MODULE
from logilab.astng.node_classes import CONST_CLS
from logilab.astng.nodes import (Module, Class, Const, const_factory, From,
Function, EmptyNode, Name, Arguments, Dict, List, Set, Tuple)
from logilab.astng.bases import Generator
from logilab.astng.manager import ASTNGManager
MANAGER = ASTNGManager()
_CONSTANTS = tuple(CONST_CLS) # the keys of CONST_CLS eg python builtin types
def _attach_local_node(parent, node, name):
node.name = name # needed by add_local_node
parent.add_local_node(node)
_marker = object()
def attach_dummy_node(node, name, object=_marker):
"""create a dummy node and register it in the locals of the given
node with the specified name
"""
enode = EmptyNode()
enode.object = object
class UCRBuilder(ConfigurationMixIn,DuckTypeHandler):
# generate XML, describing classes of project
options = OPTIONS
# criteria for duck typing
_criteria = None
_out_file = None
_project = None
_good_gettatr = 0
_bad_gettatr = 0
# numbers of "ducks" in project (for complexity estimation)
_all_ducks = 0
# numbers of classes in project (for complexity estimation)
_all_classes = 0
# _found_ducks = 0
_prob_used_classes = None
_dbg_assattr_parents = None
_list_attrs = [attr for attr in dir([]) if not re.search('\A(?!_)',attr)]
_list_methods = [attr for attr in dir([]) if re.search('\A(?!_)',attr)]
_dict_attrs = [attr for attr in dir({}) if not re.search('\A(?!_)',attr)]
_dict_methods = [attr for attr in dir({}) if re.search('\A(?!_)',attr)]
_tuple_attrs = [attr for attr in dir(()) if not re.search('\A(?!_)',attr)]
_tuple_methods = [attr for attr in dir(()) if re.search('\A(?!_)',attr)]
_attr_iteration_cycles = 0
_treshold = None
_add_value = None
def __init__(self, args,criteria='default',out_file='test.xml',treshold=None,add_value=False):
ConfigurationMixIn.__init__(self, usage=__doc__)
self._add_value = add_value
self._project = args[0]
self._treshold = treshold
self._out_file = out_file
self._criteria = criteria
self._dbg_assattr_parents = set([])
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
# Check if object is of standard complex type(dict, tuple or list)
def _check_complex_type(self,attrs,methods):
if(all(meth in self._list_methods for meth in methods) and
all(attr in self._list_attrs for attr in attrs)):
return 'List'
elif(all(meth in self._dict_methods for meth in methods) and
all(attr in self._dict_attrs for attr in attrs)):
return 'Dict'
elif(all(meth in self._tuple_methods for meth in methods) and
all(attr in self._tuple_attrs for attr in attrs)):
return 'Tuple'
return None
def get_duck_signature(self,duck):
if(duck['complex_type']):
if duck.has_key('element_signature'):
# search for class of element is needed
return set(duck['element_signature']['attrs'].keys()),set(duck['element_signature']['methods'].keys())
return set(duck['attrs'].keys()), set(duck['methods'].keys())
def run(self, args):
"""checking arguments and run project"""
if not args:
print self.help()
return
project = self.manager.project_from_files(args, astng_wrapper)
self.project = project
linker = ClassIRLinker(project)
linker.visit(project)
if self._criteria == 'capacity':
found_ducks = {}
bad_ducks = {}
prob_used_classes = {}
for t in numpy.arange(self._treshold,1,0.05):
found_ducks[t] = 0
bad_ducks[t] = 0
prob_used_classes[t] = set([])
else:
prob_used_classes = set([])
bad_ducks = 0
found_ducks = 0
ducks_num = len(list(linker.get_ducks()))
count = 1
dbg = set([])
empty_ducks = 0
""" Handle "duck" information and generate information about types """
for current_class in linker.get_classes():
for duck in current_class.cir_ducks.keys():
print "Processing ", count, " duck of ",ducks_num
# print duck,current_class.cir_ducks[duck]
count +=1
duck_attrs, duck_methods = self.get_duck_signature(current_class.cir_ducks[duck])
""" ignore empty ducks """
if((not duck_attrs) and (not duck_methods)):
empty_ducks+=1
continue
if not hasattr(current_class.cir_ducks[duck], 'complex_type'):
""" if duck is not detected as complex type on previous stage (according to [],{} etc. usage)
we need to check its methods and fields """
complex_type = self._check_complex_type(duck_attrs, duck_methods)
if(complex_type):
current_class.cir_ducks[duck]['complex_type'] = complex_type
if self._criteria == 'capacity':
for t in found_ducks.keys():
found_ducks[t]+=1
else:
found_ducks+=1
continue
if(self._criteria=='capacity'):
''' Results of candidate class search will be saved for different thresholds '''
duck_found = {}
for t in numpy.arange(self._treshold,1,0.05):
duck_found[t] = False
else:
duck_found = False
for field_candidate in linker.get_classes():
result = self.check_candidate(duck_attrs, duck_methods, field_candidate,self._criteria)
if self._criteria == 'capacity':
if(result>= self._treshold):
current_class.cir_ducks[duck]['type'].append(field_candidate)
if self._add_value:
''' save value for candidate '''
if current_class.cir_ducks[duck].has_key('type_values'):
current_class.cir_ducks[duck]['type_values'][field_candidate.cir_uid]=result
else:
current_class.cir_ducks[duck]['type_values']={field_candidate.cir_uid:result}
for t in duck_found.keys():
''' Save probably used classes for different thresholds '''
if(result>=t):
prob_used_classes[t] |= set([field_candidate.cir_uid])
duck_found[t] = True
else:
if(result):
current_class.cir_ducks[duck]['type'].append(field_candidate)
prob_used_classes |= set([field_candidate.cir_uid])
''' check if duck not found at all '''
if self._criteria =='capacity':
for t in duck_found.keys():
if(not duck_found[t]):
bad_ducks[t] += 1
else:
found_ducks[t]+=1
else:
if(not duck_found):
bad_ducks += 1
else:
found_ducks+=1
# empty_ducks = len(list(linker.get_empty_ducks()))
# print len(dbg)
# print dbg
print "Project - ",self._project
print "Duck typing criteria - ",self._criteria
print "Numbers of classes: ",len(list(linker.get_classes()))
print "Numbers of ducks(non-empty): ", linker.get_ducks_count()-empty_ducks
print "Numbers of ducks with complex type: ", len(list(linker.get_complex_ducks()))
if self._criteria == 'capacity':
if(linker.get_ducks_count()!=empty_ducks):
b = found_ducks.keys()
for t in sorted(found_ducks.keys()):
print t,"found ducks: ",found_ducks[t], " percentage from non-empty ducks: ",round(100*float(found_ducks[t])/(linker.get_ducks_count()-empty_ducks),1), " %"
if(linker.get_attrs_count()!=0):
for t in sorted(found_ducks.keys()):
print t,"Numbers of all attributes in project: ", linker.get_attrs_count(), " percentage of found attrs: ",round(100*float(found_ducks[t])/linker.get_attrs_count(),1), " %"
if(len(list(linker.get_classes()))!=0):
for t in sorted(found_ducks.keys()):
print t,"Probably used (as field) classes: ",len(prob_used_classes[t])," percentage: ",round(100*float(len(prob_used_classes[t]))/len(list(linker.get_classes())),1), " %"
else:
if(linker.get_ducks_count()!=empty_ducks):
print "Found ducks: ",found_ducks, " percentage from non-empty ducks: ",round(100*float(found_ducks)/(linker.get_ducks_count()-empty_ducks),1), " %"
if(linker.get_attrs_count()!=0):
print "Numbers of all attributes in project: ", linker.get_attrs_count(), " percentage of found attrs: ",round(100*float(found_ducks)/linker.get_attrs_count(),1), " %"
if(len(list(linker.get_classes()))!=0):
print "Probably used (as field) classes: ",len(prob_used_classes)," percentage: ",round(100*float(len(prob_used_classes))/len(list(linker.get_classes())),1), " %"
# result XML generation
mapper = {}
root = etree.Element("Classes")
for obj in linker.get_classes():
self._all_classes +=1
node = etree.Element("Class",name=obj.name,fromlineno=str(obj.fromlineno),col_offset=str(obj.col_offset),id=str(obj.cir_uid),label=obj.root().name)
mapper[obj] = node
root.append(node)
for attrname in obj.ucr_attrs:
attr_node = etree.Element('Attr',name=attrname)
mod_node = etree.Element('Modifier',name=get_visibility(attrname))
attr_node.append(mod_node)
node.append(attr_node)
if(attrname in obj.cir_ducks):
if obj.cir_ducks[attrname]['complex_type']:
for prob_type in obj.cir_ducks[attrname]['type']:
attr_node.append(etree.Element('AggregatedType',name=str(obj.cir_ducks[attrname]['complex_type']),element_type=prob_type.name,element_id=str(prob_type.cir_uid)))
else:
for prob_type in obj.cir_ducks[attrname]['type']:
if(obj.cir_ducks[attrname].has_key('type_values')):
common_type_node = etree.Element('CommonType',
name=prob_type.name,
id=str(prob_type.cir_uid),
type_value=str(obj.cir_ducks[attrname]['type_values'][prob_type.cir_uid]))
else:
common_type_node = etree.Element('CommonType',
name=prob_type.name,
id=str(prob_type.cir_uid))
attr_node.append(common_type_node)
for meth in linker.get_methods(obj):
meth_node = etree.Element('Method',name=meth.name)
meth_node.set("fromlineno",str(meth.fromlineno))
meth_node.set("col_offset",str(meth.col_offset))
mod_node = etree.Element('Modifier',name=get_visibility(meth.name))
meth_node.append(mod_node)
""" This is needed for some native libs(pyx) """
if(meth.args.args == None):
continue
for arg in meth.args.args:
# ignore self arg
if not arg.name == 'self':
meth_node.append(etree.Element('Arg',name=arg.name))
node.append(meth_node)
for rel in linker.get_inheritances():
mapper[rel[0]].append(etree.Element('Parent',name=rel[1].name,id=str(rel[1].cir_uid)))
print "Writing ", self._out_file
f = open(self._out_file,'w')
f.write(etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=True))
f.close()
def setUp(self):
self.manager = ASTNGManager()
self.manager.astng_cache.clear()
def setUp(self):
self.manager = ASTNGManager()
self.builder = ASTNGBuilder(self.manager)
self.manager.astng_cache.clear()
class LogilabUCRBuilder(ConfigurationMixIn):
''' generate XML, describing classes of project '''
options = OPTIONS
_good_gettatr = 0
_bad_gettatr = 0
# numbers of "ducks" in project (for complexity estimation)
_all_ducks = 0
# numbers of classes in project (for complexity estimation)
_all_classes = 0
_process_candidates = False
_ducks_count = 0
_found_ducks = 0
_prob_used_classes = None
_all_attrs_num = 0
_complex_ducks = 0
_assigned_ducks = 0
_dbg_assattr_parents = None
_list_attrs = [attr for attr in dir([]) if not re.search('\A(?!_)',attr)]
_list_methods = [attr for attr in dir([]) if re.search('\A(?!_)',attr)]
_dict_attrs = [attr for attr in dir({}) if not re.search('\A(?!_)',attr)]
_dict_methods = [attr for attr in dir({}) if re.search('\A(?!_)',attr)]
_tuple_attrs = [attr for attr in dir(()) if not re.search('\A(?!_)',attr)]
_tuple_methods = [attr for attr in dir(()) if re.search('\A(?!_)',attr)]
_attr_iteration_cycles = 0
_processed_methods = 0
def __init__(self, args,process_candidates=False):
ConfigurationMixIn.__init__(self, usage=__doc__)
self._prob_used_classes = Set([])
self._dbg_assattr_parents = Set([])
self._process_candidates = process_candidates
insert_default_options()
self.manager = ASTNGManager()
self.register_options_provider(self.manager)
args = self.load_command_line_configuration()
self.run(args)
""" Extract information about class signature - attrs and methods, which it have """
def _compute_signature(self,node):
if(hasattr(node, "csu_complete_signatures")):
# node have been processed already
return node.csu_complete_signatures
else:
node.csu_complete_signatures={}
node.csu_complete_signatures['Attrs'] = Set([re.search('[^ :]*',attr).group(0) for attr in node.attrs])
node.csu_complete_signatures['Methods'] = Set([meth.name for meth in node.methods])
# class without parents
if not hasattr(node, "csu_parents"):
node.csu_parents = []
parents = node.csu_parents
for parent in parents:
parent_signature = self._compute_signature(parent)
# append all parents signatures
node.csu_complete_signatures['Attrs'] |= parent_signature['Attrs']
node.csu_complete_signatures['Methods'] |= parent_signature['Methods']
return node.csu_complete_signatures
""" Check body of cycle, which iterating over class's field"""
def _check_cycle(self,node,iter_name,attr,duck_dict):
if isinstance(node, Getattr):
if(node.expr.as_string()==iter_name):
if(not duck_dict[attr].has_key('element_signature')):
duck_dict[attr]['element_signature']={'attrs':Set([]),'methods':Set([])}
if isinstance(node.parent,CallFunc):
duck_dict[attr]['element_signature']['methods'].add(node.attrname)
else:
duck_dict[attr]['element_signature']['attrs'].add(node.attrname)
for child in node.get_children():
duck_dict = self._check_cycle(child,iter_name,attr,duck_dict)
return duck_dict
""" Extract information about class fields usage """
def _extract_duck_info(self,node,attrs,duck_dict=None):
if(duck_dict is None):
duck_dict = {}
if isinstance(node, Getattr):
if(node.expr.as_string()=="self"):
if isinstance(node.parent, For):
if(not duck_dict.has_key(node.attrname)):
self._ducks_count +=1
duck_dict[node.attrname] = {'attrs':Set([]),'methods':Set([]),'type':[],'complex_type':'Unknown','assigned':False}
self._attr_iteration_cycles +=1
if isinstance(node.parent.target, AssName):
print node.parent.as_string()
for body in node.parent.body:
duck_dict = self._check_cycle(body,node.parent.target.name,node.attrname,duck_dict)
if(node.attrname not in attrs):
#print node.attrname,node.parent, node.fromlineno, node.root()
#print attrs
self._bad_gettatr+=1
else:
self._good_gettatr+=1
# if additional info about attr's field may be obtained
if isinstance(node.parent, Getattr):
#init dict for attr
if(not duck_dict.has_key(node.attrname)):
self._ducks_count +=1
duck_dict[node.attrname] = {'attrs':Set([]),'methods':Set([]),'type':[],'complex_type':None,'assigned':False}
if isinstance(node.parent.parent,CallFunc):
#we get info about attr's method
duck_dict[node.attrname]['methods'].add(node.parent.attrname)
else:
#we get info about attr's attr
duck_dict[node.attrname]['attrs'].add(node.parent.attrname)
# attr of complex type (list, dict, tuple etc.)
elif isinstance(node.parent, Subscript):
if(not duck_dict.has_key(node.attrname)):
self._ducks_count +=1
duck_dict[node.attrname] = {'attrs':Set([]),'methods':Set([]),'type':[],'complex_type':'Unknown','assigned':False}
else:
duck_dict[node.attrname]['complex_type'] = 'Unknown'
if(isinstance(node.parent.parent,Getattr)):
# get some info about element of complex type
if(not duck_dict[node.attrname].has_key('element_signature')):
duck_dict[node.attrname]['element_signature']={'attrs':Set([]),'methods':Set([])}
if isinstance(node.parent.parent.parent,CallFunc):
duck_dict[node.attrname]['element_signature']['methods'].add(node.parent.parent.attrname)
else:
duck_dict[node.attrname]['element_signature']['attrs'].add(node.parent.parent.attrname)
elif isinstance(node, AssAttr):
if(node.expr.as_string()=="self"):
if(not duck_dict.has_key(node.attrname)):
self._ducks_count +=1
self._assigned_ducks +=1
duck_dict[node.attrname] = {'attrs':Set([]),'methods':Set([]),'type':[],'complex_type':None,'assigned':True}
else:
if(not duck_dict[node.attrname]['assigned']):
duck_dict[node.attrname]['assigned'] = True
self._assigned_ducks+=1
# DEBUG
if (not node.parent.__class__.__name__ in self._dbg_assattr_parents):
self._dbg_assattr_parents |= Set([node.parent.__class__.__name__])
print node.parent.__class__.__name__
if(isinstance(node.parent, Tuple)):
print node.parent.as_string()
# DEBUG END
if(isinstance(node.parent, (Assign,AugAssign))):
if(isinstance(node.parent.value, (Tuple,Dict,List))):
duck_dict[node.attrname]['complex_type'] = node.parent.value.__class__.__name__
for child in node.get_children():
duck_dict = self._extract_duck_info(child,attrs,duck_dict)
return duck_dict
# Check if object is of standard complex type(dict, tuple or list)
def _check_complex_type(self,attrs,methods):
if(all(meth in self._list_methods for meth in methods) and
all(attr in self._list_attrs for attr in attrs)):
return 'List'
elif(all(meth in self._dict_methods for meth in methods) and
all(attr in self._dict_attrs for attr in attrs)):
return 'Dict'
elif(all(meth in self._tuple_methods for meth in methods) and
all(attr in self._tuple_attrs for attr in attrs)):
return 'Tuple'
return None
def run(self, args):
"""checking arguments and run project"""
if not args:
print self.help()
return
project = self.manager.project_from_files(args, astng_wrapper)
self.project = project
linker = NoInferLinker(project, tag=True)
handler = DiadefsHandler(self.config)
diadefs = handler.get_diadefs(project, linker)
# Add inheritance information to nodes
# csu_parents will contain links to all parents of class
for rel in diadefs[-1].relationships['specialization']:
if hasattr(rel.from_object, "csu_parents"):
rel.from_object.csu_parents.append(rel.to_object)
else:
rel.from_object.csu_parents=[rel.to_object]
bad_ducks = 0
empty_ducks = 0
# First pass for collecting "duck" information about fields
for obj in diadefs[-1].objects:
self._compute_signature(obj)
attr_names = [re.search('[^ :]*',s).group(0) for s in obj.attrs]
self._all_attrs_num += len(Set(attr_names))
attr_names+= [m.name for m in obj.methods]
duck_dict = None
for meth in obj.methods:
# check self access in method and generate information about class attrs
if(self._process_candidates):
duck_dict = self._extract_duck_info(meth,attr_names,duck_dict)
self._processed_methods += 1
# add duck information to classes
obj.ducks=duck_dict
successes = 0
#Second pass for processing "duck" information and generate information about types
for current_class in diadefs[-1].objects:
if (current_class.ducks is None):
continue
for duck in current_class.ducks.keys():
if(current_class.ducks[duck]['complex_type']):
self._complex_ducks +=1
#self._found_ducks+=1
# duck is complex type, nothing to do with it
# TODO recursively complex types
if current_class.ducks[duck].has_key('element_signature'):
# search for class of element is needed
duck_attrs = current_class.ducks[duck]['element_signature']['attrs']
duck_methods = current_class.ducks[duck]['element_signature']['methods']
else:
# duck of complex type and no duck info about element
empty_ducks += 1
continue
else:
duck_attrs = current_class.ducks[duck]['attrs']
duck_methods = current_class.ducks[duck]['methods']
# ignore empty ducks
if((not duck_attrs) and (not duck_methods)):
empty_ducks += 1
continue
duck_found = False
for field_candidate in diadefs[-1].objects:
complex_type = self._check_complex_type(duck_attrs, duck_methods)
if(complex_type):
#DEBUG
if(current_class.ducks[duck]['complex_type']):
if((current_class.ducks[duck]['complex_type'] != complex_type)
and (current_class.ducks[duck]['complex_type'] !='Unknown')):
print current_class.ducks[duck]['complex_type'], complex_type
#END DEBUG
current_class.ducks[duck]['complex_type'] = complex_type
if(not duck_found):
self._found_ducks+=1
duck_found = True
if(all(attr in field_candidate.csu_complete_signatures['Attrs'] for attr in duck_attrs) and all(method in field_candidate.csu_complete_signatures['Methods'] for method in duck_methods)):
current_class.ducks[duck]['type'].append(field_candidate)
successes += 1
self._prob_used_classes |= Set([field_candidate.fig_id])
if(not duck_found):
self._found_ducks+=1
duck_found = True
#check if duck not found at all
if(not duck_found):
bad_ducks += 1
print "Bad duck - ",duck_attrs, duck_methods
print "Bad ducks ", bad_ducks
print "Empty ducks ", empty_ducks
print "Numbers of ducks: ", self._ducks_count
print "Numbers of ducks with assignment in class: ", self._assigned_ducks
print "Numbers of ducks with complex type: ", self._complex_ducks
print "Found ducks: ",self._found_ducks, " percentage: ",round(100*float(self._found_ducks)/self._ducks_count,1), " %"
print "Numbers of all attributes in project: ", self._all_attrs_num, " percentage of found attrs: ",round(100*float(self._found_ducks)/self._all_attrs_num,1), " %"
print "Numbers of classes: ",len(diadefs[-1].objects)
print "Probably used (as field) classes: ",len(self._prob_used_classes)," percentage: ",round(100*float(len(self._prob_used_classes))/len(diadefs[-1].objects),1), " %"
print "Processed methods: ", self._processed_methods
""" result XML generation """
mapper = {}
root = etree.Element("Classes")
for obj in diadefs[-1].objects:
self._all_classes +=1
node = etree.Element("Class",name=obj.title,id=str(obj.fig_id),label=obj.node.root().name)
mapper[obj] = node
root.append(node)
for attrname in Set([re.search('[^ :]*',attr).group(0) for attr in obj.attrs]):
attr_node = etree.Element('Attr',name=attrname,modifier='public')
node.append(attr_node)
if(obj.ducks and (attrname in obj.ducks)):
if obj.ducks[attrname]['complex_type']:
for prob_type in obj.ducks[attrname]['type']:
attr_node.append(etree.Element('AggregatedType',type=str(obj.ducks[attrname]['complex_type']),element=prob_type.title,id=str(prob_type.fig_id)))
else:
for prob_type in obj.ducks[attrname]['type']:
attr_node.append(etree.Element('CommonType',name=prob_type.title,id=str(prob_type.fig_id)))
for meth in obj.methods:
meth_node = etree.Element('Method',name=meth.name,modifier='public')
# This is needed for some native libs(pyx)
if(meth.args.args == None):
continue
for arg in meth.args.args:
# ignore self arg
if not arg.name == 'self':
meth_node.append(etree.Element('Arg',name=arg.name))
node.append(meth_node)
for rel in diadefs[-1].relationships['specialization']:
mapper[rel.from_object].append(etree.Element('Parent',name=rel.to_object.title,id=str(rel.to_object.fig_id)))
f = open('test.xml','w')
f.write(etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=True))
f.close()
print len(diadefs[-1].relationships['specialization'])
#print self._good_gettatr,self._bad_gettatr
#print self._all_ducks
#print self._all_classes
"""
__docformat__ = "restructuredtext en"
import sys
from os.path import abspath
from inspect import (getargspec, isdatadescriptor, isfunction, ismethod,
ismethoddescriptor, isclass, isbuiltin)
from logilab.astng.node_classes import CONST_CLS
from logilab.astng.nodes import (Module, Class, Const, const_factory, From,
Function, EmptyNode, Name, Arguments, Dict,
List, Set, Tuple)
from logilab.astng.bases import Generator
from logilab.astng.manager import ASTNGManager
MANAGER = ASTNGManager()
_CONSTANTS = tuple(CONST_CLS) # the keys of CONST_CLS eg python builtin types
def _attach_local_node(parent, node, name):
node.name = name # needed by add_local_node
parent.add_local_node(node)
_marker = object()
def attach_dummy_node(node, name, object=_marker):
"""create a dummy node and register it in the locals of the given
node with the specified name
class ASTNGManagerTC(TestCase):
def setUp(self):
self.manager = ASTNGManager()
self.manager.astng_cache.clear()
def test_astng_from_module(self):
import unittest
astng = self.manager.astng_from_module(unittest)
self.assertEqual(astng.pure_python, True)
import time
astng = self.manager.astng_from_module(time)
self.assertEqual(astng.pure_python, False)
def test_astng_from_class(self):
astng = self.manager.astng_from_class(int)
self.assertEqual(astng.name, 'int')
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
astng = self.manager.astng_from_class(object)
self.assertEqual(astng.name, 'object')
self.assertEqual(astng.parent.frame().name, BUILTINS_NAME)
self.failUnless('__setattr__' in astng)
def _test_astng_from_zip(self, archive):
origpath = sys.path[:]
sys.modules.pop('mypypa', None)
archive_path = join(DATA, archive)
sys.path.insert(0, archive_path)
try:
module = self.manager.astng_from_module_name('mypypa')
self.assertEqual(module.name, 'mypypa')
self.failUnless(module.file.endswith('%s/mypypa' % archive),
module.file)
finally:
# remove the module, else after importing egg, we don't get the zip
if 'mypypa' in self.manager.astng_cache:
del self.manager.astng_cache['mypypa']
del self.manager._mod_file_cache[('mypypa', None)]
if archive_path in sys.path_importer_cache:
del sys.path_importer_cache[archive_path]
sys.path = origpath
def test_astng_from_module_name_egg(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.egg')
def test_astng_from_module_name_zip(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.zip')
def test_from_directory(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, 'data')
self.assertEqual(obj.name, 'data')
self.assertEqual(obj.path, join(DATA, '__init__.py'))
def test_project_node(self):
obj = self.manager.project_from_files([DATA], _silent_no_wrap, 'data')
expected = set([
'SSL1', '__init__', 'all', 'appl', 'format', 'module', 'module2',
'noendingnewline', 'nonregr', 'notall'
])
expected = [
'data', 'data.SSL1', 'data.SSL1.Connection1', 'data.absimport',
'data.all', 'data.appl', 'data.appl.myConnection', 'data.email',
'data.format', 'data.module', 'data.module2',
'data.noendingnewline', 'data.nonregr', 'data.notall'
]
self.assertListEqual(sorted(k for k in obj.keys()), expected)
def test_do_not_expose_main(self):
obj = self.manager.astng_from_module_name('__main__')
self.assertEqual(obj.name, '__main__')
self.assertEqual(obj.items(), [])
# make all node classes accessible from astng package
from logilab.astng.nodes import *
# trigger extra monkey-patching
from logilab.astng import inference
# more stuff available
from logilab.astng import raw_building
from logilab.astng.bases import YES, Instance, BoundMethod, UnboundMethod
from logilab.astng.node_classes import are_exclusive, unpack_infer
from logilab.astng.scoped_nodes import builtin_lookup
# make a manager instance (borg) as well as Project and Package classes
# accessible from astng package
from logilab.astng.manager import ASTNGManager, Project
MANAGER = ASTNGManager()
del ASTNGManager
# load brain plugins
from os import listdir
from os.path import join, dirname
BRAIN_MODULES_DIR = join(dirname(__file__), 'brain')
if BRAIN_MODULES_DIR not in sys.path:
# add it to the end of the list so user path take precedence
sys.path.append(BRAIN_MODULES_DIR)
# load modules in this directory
for module in listdir(BRAIN_MODULES_DIR):
if module.endswith('.py'):
__import__(module[:-3])
class ASTNGManagerTC(unittest.TestCase):
def setUp(self):
self.manager = ASTNGManager(borg=False)
def test_astng_from_module(self):
astng = self.manager.astng_from_module(unittest)
self.assertEquals(astng.pure_python, True)
import time
astng = self.manager.astng_from_module(time)
self.assertEquals(astng.pure_python, False)
def test_astng_from_class(self):
astng = self.manager.astng_from_class(file)
self.assertEquals(astng.name, 'file')
self.assertEquals(astng.parent.frame().name, '__builtin__')
astng = self.manager.astng_from_class(object)
self.assertEquals(astng.name, 'object')
self.assertEquals(astng.parent.frame().name, '__builtin__')
self.failUnless('__setattr__' in astng)
def _test_astng_from_zip(self, archive):
origpath = sys.path[:]
sys.modules.pop('mypypa', None)
sys.path.insert(0, join(dirname(__file__), 'data', archive))
try:
module = self.manager.astng_from_module_name('mypypa')
self.assertEquals(module.name, 'mypypa')
self.failUnless(module.file.endswith('%s/mypypa' % archive),
module.file)
finally:
sys.path = origpath
def test_astng_from_module_name_egg(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.egg')
def test_astng_from_module_name_zip(self):
self._test_astng_from_zip('MyPyPa-0.1.0-py2.5.zip')
def test_from_directory(self):
obj = self.manager.from_directory('data')
self.assertEquals(obj.name, 'data')
self.assertEquals(obj.path, join(os.getcwd(), 'data'))
def test_package_node(self):
obj = self.manager.from_directory('data')
expected_short = ['SSL1', '__init__', 'all', 'appl', 'format', 'module', 'module2',
'noendingnewline', 'nonregr', 'notall']
expected_long = ['SSL1', 'data', 'data.all', 'appl', 'data.format', 'data.module',
'data.module2', 'data.noendingnewline', 'data.nonregr',
'data.notall']
self.assertEquals(obj.keys(), expected_short)
self.assertEquals([m.name for m in obj.values()], expected_long)
self.assertEquals([m for m in list(obj)], expected_short)
self.assertEquals([(name, m.name) for name, m in obj.items()],
zip(expected_short, expected_long))
self.assertEquals([(name, m.name) for name, m in obj.items()],
zip(expected_short, expected_long))
self.assertEquals('module' in obj, True)
self.assertEquals(obj.has_key('module'), True)
self.assertEquals(obj.get('module').name, 'data.module')
self.assertEquals(obj['module'].name, 'data.module')
self.assertEquals(obj.get('whatever'), None)
self.assertEquals(obj.fullname(), 'data')
