def get_pyclbr ( self ):
if not self.cache:
return pyclbr.readmodule( self.full_name, [ self.parent.path ] )
pyclbr_name = join( self.parent.path, self.name + '.pyclbr' )
if exists( pyclbr_name ):
pyclbr_stat = stat( pyclbr_name )
py_stat = stat( self.path )
if pyclbr_stat.st_mtime >= py_stat.st_mtime:
try:
file = open( pyclbr_name, 'rb' )
try:
dic = load( file )
finally:
file.close()
return dic
except:
pass
dic = pyclbr.readmodule( self.full_name, [ self.parent.path ] )
try:
file = open( pyclbr_name, 'wb' )
try:
dump( dic, file )
finally:
file.close()
except:
pass
return dic
def procesar_modulo(file_handler, modulenames, path):
# para guardar las declaraciones de herencia multiple pendientes: (derivada, base)
hm = []
#path, name = os.path.split(modname)
clsdict = {}
for modname in modulenames:
if path:
new_classes = pyclbr.readmodule(modname, [path])
else:
new_classes = pyclbr.readmodule(modname)
clsdict.update(new_classes)
clslist = clsdict.values()
for cls in clslist:
cls.super = sorted(s.name if hasattr(s, 'name') else s for s in cls.super)
# las bases primero, queda mejor :)
clslist.sort(key=lambda c:(len(c.super), c.super))
for cls in clslist:
if cls.name not in clsdict:
continue
procesar_cls(file_handler, cls, clsdict, hm)
# herencia multiple pendiente
# (trato de mostrarla tan pronto como sea posible)
while hm:
subcls, base = hm.pop(0)
file_handler.write("%s -> %s\n" % (subcls, base))
def submodule_hook_by_attr(self, submodule, prefix, name=None):
try:
_packages = pyclbr.readmodule(submodule.__name__)
except:
sys.path.append(self._env.get('PWD'))
#_packages = pyclbr.readmodule(submodule.__name__)
_packages = pyclbr.readmodule('.'.join(
submodule.__name__.split('.')[1:]))
self._cls_browse.update(_packages)
packages = {}
for cls_name, cls in _packages.items():
obj = getattr(submodule, cls_name)
if not hasattr(obj, self.attr_filter) or is_abstract(obj):
#if not any([hasattr(obj, attr) for attr in self.attr_filter]) or is_abstract(obj):
continue
if self.shrink_module_name:
prefix = prefix.split('.')[0]
if prefix:
name = prefix + '.' + cls_name.lower()
else:
name = cls_name.lower()
packages[name] = obj
return packages
def get_pyclbr(self):
if not self.cache:
return pyclbr.readmodule(self.full_name, [self.parent.path])
pyclbr_name = join(self.parent.path, self.name + '.pyclbr')
if exists(pyclbr_name):
pyclbr_stat = stat(pyclbr_name)
py_stat = stat(self.path)
if pyclbr_stat.st_mtime >= py_stat.st_mtime:
try:
file = open(pyclbr_name, 'rb')
try:
dic = load(file)
finally:
file.close()
return dic
except:
pass
dic = pyclbr.readmodule(self.full_name, [self.parent.path])
try:
file = open(pyclbr_name, 'wb')
try:
dump(dic, file)
finally:
file.close()
except:
pass
return dic
def create_random_armies(self):
self.get_config_battle()
unit_types = pyclbr.readmodule('unit')
base_types = pyclbr.readmodule('base')
for base in base_types:
unit_types.pop(base, 'None')
unit_types = tuple(unit_types.keys())
count_arm = random.randrange(int(self.rules['armies']),
int(self.rules['armies']) * 2)
# print('COUNT_ARMIES', count_arm)
for number_arm in range(count_arm):
army = []
name = 'Army ' + str(number_arm + 1)
strategy = random.choice(self.rules['strategies'].split('|'))
count_squads = random.randrange(int(self.rules['squads']),
int(self.rules['squads']) * 2)
for number_squad in range(count_squads):
squad = {}
for unit_type in unit_types:
squad[unit_type] = random.randrange(
int(self.rules['minunits']),
int(self.rules['maxunits']))
army.append(squad)
arm = Army(name, strategy, army)
self.armies.append(arm)
def register_all_models(module=None, path=None):
""" This function registers all modules in with the django admin.
The module name should be a string, and defaults to 'models' and the path can be a string, list or tuple
If you include the admin.ModelAdmin in the models.py module with the same name + Admin
then it will register them too. Example if the model class is Pizza, then the admin model
class would be PizzaAdmin """
if module is None:
module = 'models'
if path is None:
path = os.path.dirname(os.path.abspath(__file__))
classes = pyclbr.readmodule(module, [path])
elif type(path) is str:
classes = pyclbr.readmodule(module, [path])
else:
classes = pyclbr.readmodule(module, path)
# first make a list of string only parents
for model in classes:
if classes[model].super[0] in classes.values():
classes[model].super = classes[model].super[0].super
# make a list of admin classes
admin_classes = []
for model in classes:
for superclass in classes[model].super:
try:
if re.search('admin.ModelAdmin', superclass):
admin_classes.append(model)
except:
pass
for model in classes:
# now the dirty part, check that the models are classes that inherit from models.Model
# if this inhertance is not explicit in the class call it will not be registered
for superclass in classes[model].super:
try:
if re.search('models.Model', superclass):
try:
# Check to see if the modelNameAdmin is in the list of admin classes
test_name = model + 'Admin'
if test_name in admin_classes:
exec('from %s import %s,%s' %
(module, model, test_name))
exec('admin.site.register(%s,%s)' %
(model, test_name))
else:
# this could be a from module import * above this loop
exec('from %s import %s' % (module, model))
exec('admin.site.register(%s)' % model)
except:
raise
except:
pass
def discover_namespace_plugins(namespace_package):
"""Get the contents of a namespace package.
Args:
Get the `Item` defined in a namespace package
"""
packages = {}
logger.debug("Looking for Item in: %s", namespace_package.__name__)
for _, name, ispkg in pkgutil.iter_modules(
namespace_package.__path__, f"{namespace_package.__name__}."
):
if ispkg:
packages = {
**packages,
**discover_namespace_plugins(importlib.import_module(name)),
}
else:
_classes = pyclbr.readmodule(name)
for _class in _classes.values():
logger.debug("Found class: %s", _class.name)
for super_class in _class.super:
if super_class == "Item":
_class_name = f"{name}.{_class.name}"
logger.debug("Adding Item: %s", _class_name)
packages[_class.name] = getattr(
importlib.import_module(name), _class.name
)
break
return packages
def load_service(self, service_path):
"""
Load the service class pointed to by `service_path` and return an
object of the service.
"""
log.debug("Loading service class in module '{0}'".format(service_path))
module_name = os.path.splitext(os.path.basename(service_path))[0]
module_dir = os.path.dirname(service_path)
module = (os.path.splitext(service_path)[0]).replace('/', '.')
# Figure out the class name for the service
module_classes = pyclbr.readmodule(module_name, [module_dir])
# log.debug("Module {0} classes: {1}".format(module_name, module_classes))
service_class_name = None
for c in module_classes.iterkeys():
if c.lower() == ('{0}service'.format(module_name)).lower():
service_class_name = c
break
# log.debug('service_class_name: %s' % service_class_name)
# Import the service module and instantiate the service class
service = None
if service_class_name:
log.debug("Importing service name {0} as module {1}".format(
service_class_name, module))
service_module = importlib.import_module(module)
service = getattr(service_module, service_class_name)(self.app)
else:
log.warning("Could not extract service class name from module at {0}"
.format(service_path))
# log.debug("Loaded service {0}".format(service))
return service
def GetModelsOfApp(self, AppDir):
modelsPath = Path(AppDir).name
#print ("modelsPath " +modelsPath )
modelsApp = modelsPath+".models"
Classes = pyclbr.readmodule(modelsApp ).keys()
#Classes = pyclbr.readmodule(str(modelsPath.absolute())).keys()
return Classes
def getClassDefs2(path):
"""A better way?
Given a single file, find out what classes it defines.
*** Should make a "list modules used, with paths" thing.
Doc says that pyclbr.readmodule `pathList` arg "is used to augment
the value of sys.path". In fact it is checked ""before"" sys.path,
so can be used to check a specific file.
"""
import pyclbr
curDir, name = os.path.split(path)
if (not name.endswith(".py")):
raise ValueError
name = name[0:-3]
try:
classDict = pyclbr.readmodule(name, path=curDir)
except ImportError as e:
lg.vMsg(1,
" Unable to access module '%s/%s': %s" % (curDir, name, e))
return ([])
bigDict = {}
for k, v in classDict.items():
lg.vMsg(1, " %-16s %-16s %s" % (k, v.module, v.file))
if (k in bigDict):
lg.vMsg(1, " DUPLICATE %-16s %-16s %s" % (k, v.module, v.file))
bigDict[k] += "\n" + v.file
else:
bigDict[k] = v.file
return (bigDict)
def _get_classes_by_module_name(pkg_name, mod_name):
full_mod_name = mod_name
if pkg_name:
full_mod_name = '.'.join((pkg_name, mod_name))
classes = pyclbr.readmodule(full_mod_name) # a dict of 'class_name': pyclbr.Class obj
# TODO filter classes to get only subclasses of unittest.TestCase
return classes
def create_tables():
module_name = 'app.models'
module_info = pyclbr.readmodule(module_name)
db_models = []
for value in module_info.itervalues():
base = value.super[0]
if hasattr(base, 'name') and base.name == 'BaseModel':
db_models.append((value.lineno, value.name))
# it is important to preserve order as declared in module
# to prevent integrity errors
for _, model in sorted(db_models):
model_class = getattr(models, model)
try:
model_class.create_table()
term.writeLine('Table for %s created.' %
model, term.green)
except peewee.OperationalError as e:
if e.args[0] == 1050:
term.writeLine('%s already exists, skipping...' %
model, term.bold)
else:
raise e
term.writeLine('Database successfully initialized for %s '
'models.' % len(db_models), term.green)
def get_parsers_names():
"""Returns a list of the parsers names."""
name = os.path.splitext(os.path.basename(__file__))[0]
list_cls_names = (pyclbr.readmodule(name).keys())
return list_cls_names
def find_injectable_classes(search_paths, exclude_injectable_module_paths=None):
modules = set()
for path in search_paths:
for root, dirs, fnames in os.walk(path):
for fname in fnames:
if fname.endswith('.py'):
module_path = os.path.relpath(os.path.join(root, fname), path)
module = module_path.replace('/', '.')[:-3]
fpath = os.path.join(root, fname)
has_import = False
has_decorator = False
with open(fpath) as f:
for line in f:
if 'dart.context.locator' in line:
has_import = True
if '@injectable' in line:
has_decorator = True
if has_import and has_decorator:
break
if has_import and has_decorator and not path_excluded(module, exclude_injectable_module_paths):
modules.add(module)
for module in modules:
class_metadata = readmodule(module)
for class_name in class_metadata.keys():
# the line below will load the class, which causes the @injectable code to run,
# registering the class (assuming the module search was not a false positive)
locate(module + '.' + class_name)
classes_by_name = {cls.__name__: cls for cls in class_registry.classes}
for class_name in sorted(classes_by_name.keys()):
_logger.info('injectable class registered: %s' % class_name)
return classes_by_name.values()
def loadPluginsFromFolderName(self, folder_file):
pluginClass = folder_file.rstrip("/")
pluginClass_module = pluginClass.replace("/", ".")[:-3]
pluginModule = pyclbr.readmodule(pluginClass_module)
for name, Class in pluginModule.iteritems():
self.loadPlugin(name, pluginClass_module)
def toPlantUML(module, outputFile):
if os.path.isfile(module):
module = os.path.splitext(os.path.basename(module))[0]
with open(outputFile, "w") as f:
f.write(STARTUML)
f.write(STYLE)
title = TITLE.format(package=module)
f.write(title)
classDescriptors = pyclbr.readmodule(module)
for className, classData in classDescriptors.items():
child = className
methods = sorted([m + "()" for m in classData.methods])
parents = [
p.name if hasattr(p, "name") else str(p)
for p in classData.super
]
for parent in parents:
relationLine = getRelationLine(parent, child)
f.write(relationLine)
for method in methods:
methodLine = getMethodLine(child, method)
f.write(methodLine)
f.write(ENDUML)
os.system("notepad " + outputFile)
def map_tasks(pkg):
"""Map task names to their modules and classes.
The method assumes that the task name is practically identical with
the respective class name except that class name
1. ends with 'Task',
2. follows CapWord naming convention.
So task `doSomething` is expected to be implemented by the class
`DoSomethingTask`.
Parameters
----------
pkg : `str`
Package to search.
"""
tasks = {}
for _, mod, _ in pkgutil.iter_modules(pkg.__path__):
classes = pyclbr.readmodule(mod, path=pkg.__path__)
tasks.update({name: pkg.__name__ + '.' + cls.module
for name, cls in classes.items()
if (cls.module == mod and
cls.name.lower().endswith('task'))})
return {cls[0].lower() + cls[1:-4]: (mod, cls)
for cls, mod in tasks.items()}
def main(argv, failfast=False, test_labels=None):
testlist = []
for module in [name for _, name, _ in pkgutil.iter_modules([os.path.join("cms","tests")])]:
clsmembers = pyclbr.readmodule("cms.tests.%s" % module)
for clsname,cls in clsmembers.items():
testlist.append(cls)
failures = []
for cls in testlist:
for method, line in cls.methods.items():
if not method.startswith('test_'):
continue
test = '%s.%s' % (cls.name, method)
if not test_labels or filter(lambda x: test.find(x)>-1, test_labels):
print("Running ",test)
args = ['python', 'runtests.py'] + argv + [test]
p = subprocess.Popen(args, stdout = subprocess.PIPE, stderr= subprocess.PIPE)
output, error = p.communicate()
if p.returncode > 0:
print(error)
if failfast:
sys.exit(p.returncode)
else:
failures.append(test)
else:
print()
print("Result: %s" % ('FAIL' if failures else 'OK'))
print("%s Failures:" % len(failures))
for failure in failures:
print("- %s" % failure)
sys.exit(len(failures))
def available_tasks():
modules = ["preingest.tasks", "ESSArch_Core.WorkflowEngine.tests.tasks"]
tasks = []
for m in modules:
module_tasks = pyclbr.readmodule(m)
tasks = tasks + zip([m + "." + t for t in module_tasks], module_tasks)
return tasks
def findInstrumentModule(self, instrumentModuleName):
"""
Walks down the instrument directory tree, looks for the first valid instrument module with specified name instrumentModuleName,
and returns:
- a tuple (module name, filename) if a valid module was found
- or None if no valid module was found.
instrumentModuleName is a dotted module name string possibly including '.' chars; it is NOT a filename string.
The instrument module is considered as valid when it has the specified name and contains a definition for the class Instr.
"""
found = None
for (dirpath, dirnames, filenames) in os.walk(self._instrumentsRootDir): # start walking down the directory tree and at each step
# builds a list of all python file names (except _init__.py)
pyFilenames = [filename for filename in filenames if (
os.path.splitext(filename)[1] == '.py' and filename != '__init__.py')]
pyFilenames = [os.path.splitext(filename)[0] for filename in pyFilenames]
if instrumentModuleName in pyFilenames:
try:
# build the class dictionary with the pyclbr python class browser
dic = pyclbr.readmodule(instrumentModuleName, [dirpath])
# check that the module contains a class definition Instr in the file and not through an import.
if 'Instr' in dic:
path1 = os.path.realpath(dic['Instr'].file)
path2 = os.path.join(os.path.realpath(dirpath), instrumentModuleName + '.py')
if path1 == path2:
found = (dic['Instr'].module, path1)
break # stop the walk
except:
print 'an error occured when trying to read the module.'
finally:
pass
return found
def get_models() -> list:
"""
Scans `settings.apps_folder_name`.
Find `models` modules in each of them and get all attributes there.
Last step is to filter attributes to return only those,
subclassed from MongoDBModel (or timestamped version).
Used internally only by `create_indexes` function.
:return: list of user-defined models (subclassed from MongoDBModel) in apps
"""
from fastapi_contrib.db.models import MongoDBModel
apps_folder_name = settings.apps_folder_name
models = []
for app in settings.apps:
app_path = f"{apps_folder_name}/{app}"
modules = [f[1] for f in pkgutil.walk_packages(path=[app_path])]
if "models" in modules:
path_to_models = f"{apps_folder_name}.{app}.models"
try:
module_models = pyclbr.readmodule(path_to_models).keys()
except (AttributeError, ImportError):
logger.warning(
f"Unable to read module attributes in {path_to_models}"
)
continue
mudule = importlib.import_module(
f"{apps_folder_name}.{app}.models"
)
models.extend([getattr(mudule, model) for model in module_models])
return list(filter(lambda x: issubclass(x, MongoDBModel), models))
def __init__(self, categories, parent=None):
super(_LgsipGatesWidget, self).__init__(parent)
self.setRootIsDecorated(False)
self.setItemDelegate(_Delegate())
self.setIndentation(0)
self.setFixedWidth(110)
self.setExpandsOnDoubleClick(False)
self.setDragDropMode(self.DragOnly)
self.header().close()
self.itemClicked.connect(self._expandCollapse)
for name, id in categories:
module = self._module + id.lower()
item = QtGui.QTreeWidgetItem([name])
self.addTopLevelItem(item)
g = pyclbr.readmodule(module)
for gate in g.keys():
if gate[0] != '_':
subitem = QtGui.QTreeWidgetItem()
item.addChild(subitem)
module_ = __import__(module, globals(), locals(), gate)
widget = getattr(module_, gate)()
pixmap = self.createPixmap(widget)
subitem.setData(0, 666, pixmap)
subitem.setData(0, 667, gate)
subitem.setData(0, 668, module)
self.expandAll()
def create(config, logger = None, options = None):
''' instantiate notifiers '''
if logger:
logger.info("loading notifiers ...")
if not config.has_option('notification', 'plugins'):
return []
plugins = config.get('notification', 'plugins')
notifiers = []
for modname in [ p.strip() for p in plugins.split(',') if p.strip() ]:
mod = __import__(modname, globals(), locals(), [modname], -1)
for clzname in pyclbr.readmodule(mod.__name__).keys():
if clzname == 'Notifier':
continue
clz = getattr(mod, clzname)
if issubclass(clz, Notifier):
if logger:
logger.info("instantiating notifier: {0}".format(clzname))
inits = dict(config.items(clzname)) if config.has_section(clzname) else {}
if options:
inits.update( options )
notifier = clz(**inits)
notifier.use_logger(logger)
notifiers.append(notifier)
return notifiers
def register_issues():
klasses = [
globals().get(a.name)
for a in pyclbr.readmodule('reports.issues').values()
]
return {a.name: a for a in klasses if hasattr(a, 'name')}
def toPlantUML(module, outputFile):
if os.path.isfile(module):
module = os.path.splitext(os.path.basename(module))[0]
with open(outputFile, "w") as f:
f.write(STARTUML)
f.write(STYLE)
title = TITLE.format(package=module)
f.write(title)
classDescriptors = pyclbr.readmodule(module)
for className, classData in classDescriptors.items():
child = className
methods = sorted([m + "()" for m in classData.methods])
parents = [p.name if hasattr(p, "name") else str(p) for p in classData.super]
for parent in parents:
relationLine = getRelationLine(parent, child)
f.write(relationLine)
for method in methods:
methodLine = getMethodLine(child, method)
f.write(methodLine)
f.write(ENDUML)
os.system("notepad " + outputFile)
def import_classes(package,
module_name_filter=None,
class_name_filter=None,
base_class=None):
classes = []
path = importlib.import_module(package).__path__
logger.debug("Loading module in directory %s" % path)
for null, name, ispkg in pkgutil.iter_modules(
importlib.import_module(package).__path__):
if ispkg or (module_name_filter
and not re.match(module_name_filter, name)):
continue
module_name = '.'.join([package, name])
logger.debug("Importing module '%s'", module_name)
module = importlib.import_module(module_name)
for class_info in pyclbr.readmodule(module_name).values():
if class_name_filter and not re.match(class_name_filter,
class_info.name):
continue
class_ = getattr(module, class_info.name)
if base_class and not issubclass(class_, base_class):
continue
classes.append(class_)
return classes
def add_storage_adaptor(obj, storage_adapter):
available_adaptors = list(readmodule("alchemydumps.storage"))
adaptor_name = f'{storage_adapter.title()}Storage'
adaptor = [a for a in available_adaptors if a == adaptor_name][0]
base_cls = obj.__class__
base_cls_name = obj.__class__.__name__
obj.__class__ = type(base_cls_name, (base_cls, adaptor), {})
def is_vm_ill_defined(self, vm_module):
"""Check whether a virtual machine is well or ill defined.
Makes sure that selected virtual machine contains one and only one
user-defined virtual machine class, child of py3Gestalt's
'machines.VirtualMachine' class.
Args:
vm_module: Virtual Machine module to be analyzed
Returns:
True when selected module contains none or more than a unique
virtual machine. False otherwise.
"""
num_of_vm_cls = 0
for name, class_data in sorted(pyclbr.readmodule(vm_module).items(),
key=lambda x: x[1].lineno):
if class_data.super[0] == 'machines.VirtualMachine':
num_of_vm_cls += 1
if num_of_vm_cls == 0:
self.write_debugger("Error: No virtual machine defined." + '\n' +
"Select a new file.")
return True
elif num_of_vm_cls > 1:
self.write_debugger("Error: More than a unique virtual " +
"machine defined in a single file." + '\n' +
"Select a new file.")
return True
self.write_debugger("Virtual machine correctly defined.")
return False
def load_providers(cls):
# Search Provider Files in the Directory
for pfile in os.listdir('Providers'):
# Discard non-Python files
if '.py' not in pfile:
continue
# Discard provider.py and __init__.py Files
if 'provider.py' in pfile or '__init__.py' in pfile:
continue
# Remove .py extension
pfile = pfile[:-3]
# Get Classes names in Modules
for provider in pyclbr.readmodule(f'Providers.{pfile}').keys():
# Getting the module
module = importlib.import_module(f'Providers.{pfile}')
# Getting the Class
class_ = getattr(module, provider)
# Instanciate the Class
instance = class_()
# Add the Provider Instance to the Providers List
cls.providers.append(instance)
def _load_commands(self):
"""
Рефлективно загружает, инициализирует и регистрирует все команды ВК из модуля 'vkcmds'.
:return: два списка (list): (1) список экземпляров <? extends VKCommand> загруженных команд,
(2) список названий загруженных команд ДЛЯ БЕСЕД (команды ЛС не включены).
"""
cmd_submodules = dict()
# Ищем все подмодули и все классы в них без импорта самих подмодулей.
for root, dirs, files in os.walk("vkcmds", topdown=False):
abs_search_path = os.path.join(os.path.dirname(__file__), root,
'*.py')
for path in glob.glob(abs_search_path):
submodule_name = os.path.basename(path)[:-3] # -3 из-за '.py'
all_classes = pyclbr.readmodule("{0}.{1}".format(
root.replace(os.path.sep, '.'), submodule_name))
# Ищем в подмодуле класс, наследующий VKCommand.
command_classes = {
name: info
for name, info in all_classes.items()
if 'VKCommand' in info.super
}
if command_classes: # подходящий класс найден
cmd_submodules[(root.replace(os.path.sep, '.'),
submodule_name)] = command_classes
commands = [] # экземпляры классов зарегистрированных команд
chat_command_names = [
] # названия зарегистрированных команд ДЛЯ БЕСЕД (названия команд для ЛС не включены)
# Проходимся по подмодулям команд, инициализируем классы команд в них (для каждой
# команды создаётся один её экземпляр) и добавляем полученные объекты в список команд.
for submodule, cmd_classes in cmd_submodules.items():
submodule_root, submodule_name = submodule
module = __import__(f'{submodule_root}.{submodule_name}'
) # импортируем подмодуль по имени
for mod in submodule_root.split(".")[1:]:
module = getattr(module, mod) # идём до папки
submodule = getattr(module,
submodule_name) # получаем сам подмодуль
# Проходимся по всем классам команд.
for cmd_class_name in cmd_classes:
# Создаём экземпляр этого класса (инициализируем его) и добавляем в список команд.
class_instance = getattr(submodule,
cmd_class_name)(self.kristy)
cmd_label = class_instance.label
dm = class_instance.dm
commands.append(
class_instance
) # короче ты получал класс, а нужно было объект!!!
if not dm:
chat_command_names.append(cmd_label)
self.logger.info('Загружена команда ВК (%s): "%s"',
'для ЛС' if dm else 'для бесед', cmd_label)
# Возвращаем список экземпляров загруженных команд и названия этих команд.
return commands, chat_command_names
def _get_classes_by_module_name(pkg_name, mod_name):
full_mod_name = mod_name
if pkg_name:
full_mod_name = '.'.join((pkg_name, mod_name))
classes = pyclbr.readmodule(
full_mod_name) # a dict of 'class_name': pyclbr.Class obj
# TODO filter classes to get only subclasses of unittest.TestCase
return classes
def find_modules_with_super_class(pkg, super_class):
for importer, modname, ispkg in pkgutil.walk_packages(pkg.__path__):
if ispkg: continue
import_path = "%s.%s" % (pkg.__name__, modname)
module = pyclbr.readmodule(import_path)
for item, val in module.items():
if super_class.__name__ in val.super:
yield item, import_path
def find(self, path, reset=False):
py_files = self._find_modules(path, reset=reset)
for module_str in py_files:
browser = pyclbr.readmodule(module_str)
for name in browser.keys():
obj = get_module('{}'.format(module_str), name)
if issubclass(obj, self.superclass):
yield obj
def register_all_models(module=None,path=None):
""" This function registers all modules in with the django admin.
The module name should be a string, and defaults to 'models' and the path can be a string, list or tuple
If you include the admin.ModelAdmin in the models.py module with the same name + Admin
then it will register them too. Example if the model class is Pizza, then the admin model
class would be PizzaAdmin """
if module is None:
module='models'
if path is None:
path=os.path.dirname(os.path.abspath(__file__))
classes = pyclbr.readmodule(module,[path])
elif type(path) is str:
classes = pyclbr.readmodule(module,[path])
else:
classes = pyclbr.readmodule(module,path)
# first make a list of string only parents
for model in classes:
if classes[model].super[0] in classes.values():
classes[model].super=classes[model].super[0].super
# make a list of admin classes
admin_classes=[]
for model in classes:
for superclass in classes[model].super:
try:
if re.search('admin.ModelAdmin',superclass):
admin_classes.append(model)
except:pass
for model in classes:
# now the dirty part, check that the models are classes that inherit from models.Model
# if this inhertance is not explicit in the class call it will not be registered
for superclass in classes[model].super:
try:
if re.search('models.Model',superclass):
try:
# Check to see if the modelNameAdmin is in the list of admin classes
test_name=model+'Admin'
if test_name in admin_classes:
exec('from %s import %s,%s'%(module,model,test_name))
exec('admin.site.register(%s,%s)'%(model,test_name))
else:
# this could be a from module import * above this loop
exec('from %s import %s'%(module,model))
exec('admin.site.register(%s)'%model)
except:raise
except:pass
def is_python_module_name(filename):
path, name = os.path.split(filename)
try:
clsdict = pyclbr.readmodule(name, path)
except ImportError:
return False
return True
def loadCasePlugins(self):
classes = pyclbr.readmodule("motorcases")
for c in classes:
if not (c.endswith("Base")):
classobj = eval(c)
if (issubclass(classobj, TestCase)):
if not (classobj == TestCase):
classinstance = classobj(self)
def retrieve_migrations(self):
if self.migrations_module is not None:
class_names = pyclbr.readmodule(self.migrations_mstr).keys()
for name in class_names:
if name != 'Migration' and 'Migration' in name:
cobject = getattr(self.migrations_module, name)
self.migrations.append(cobject)
def init():
global pattern_class
for _,name,_ in pkgutil.iter_modules(path=patterns.__path__):
module_path = 'patterns.{}'.format(name)
module = importlib.import_module(name=module_path)
cls_list = pyclbr.readmodule(module_path)
for cls in cls_list:
pattern_class.append(getattr(module,cls))
def import_executor(
self,
folder: str,
base_folder: str,
executor: str,
libraries: List[Tuple] = None
):
sys.path.insert(0, base_folder)
spec = self._build_spec(folder)
was_installation = False
folders = [
p for p in glob(f'{folder}/*', recursive=True)
if os.path.isdir(p) and not spec.match_file(p)
]
folders += [folder]
library_names = set(n for n, v in (libraries or []))
library_versions = {n: v for n, v in (libraries or [])}
for n in library_names:
try:
version = pkg_resources.get_distribution(n).version
need_install = library_versions[n] != version
except Exception:
need_install = True
if INSTALL_DEPENDENCIES and need_install:
os.system(f'pip install {n}=={library_versions[n]}')
was_installation = True
def is_valid_class(cls: pyclbr.Class):
return cls.name == executor or \
cls.name.lower() == executor or \
to_snake(cls.name) == executor
def relative_name(path: str):
rel = os.path.relpath(path, base_folder)
parts = [str(p).split('.')[0] for p in rel.split(os.sep)]
return '.'.join(parts)
for (module_loader, module_name,
ispkg) in pkgutil.iter_modules(folders):
module = module_loader.find_module(module_name)
rel_path = os.path.relpath(
os.path.splitext(module.path)[0], base_folder
).replace('/', '.')
try:
classes = pyclbr.readmodule(rel_path, path=[base_folder])
except Exception:
continue
for k, v in classes.items():
if is_valid_class(v):
importlib.import_module(relative_name(module.path))
return True, was_installation
return False, was_installation
def get_all_curricula(message):
msg = ''
module_name = 'generate_sheet'
module_info = pyclbr.readmodule(module_name)
for item in module_info.values():
if 'default_num' in item.methods and item.module not in msg:
msg += (item.module + ', ')
msg = msg[:-2]
return msg
def _get_test_labels():
test_labels = []
for module in [name for _, name, _ in pkgutil.iter_modules([os.path.join("cms","tests")])]:
clsmembers = pyclbr.readmodule("cms.tests.%s" % module)
for clsname, cls in clsmembers.items():
for method, _ in cls.methods.items():
if method.startswith('test_'):
test_labels.append('cms.%s.%s' % (clsname, method))
return test_labels
def print_builtin_cameras():
# TODO check this maple prefix works installed other ways?
cams = pyclbr.readmodule('maple.cameras')
cams.pop('Camera')
cams.pop('CameraNotFoundError')
cams.pop('NoFrameError')
print '\n\nAvailable cameras:'
for c in cams:
print 'maple.cameras.{}'.format(c)
def get_parsers_names():
"""Returns a list of the parsers names, whith no Base classes."""
name = os.path.splitext(os.path.basename(__file__))[0]
list_cls_names = (pyclbr.readmodule(name).keys())
list_no_base_cls_names = [cls_name for cls_name in list_cls_names
if cls_name[:4] != "Base"]
return list_no_base_cls_names
def get_all_topics(message):
msg = ''
module_name = 'generate_sheet'
module_info = pyclbr.readmodule(module_name)
for item in module_info.values():
# Check if item has attribute "insert_question"
if 'default_num' in item.methods:
msg += (item.name + ', in ' + item.module + '\n')
return msg
def get_parsers_names():
"""Returns a list of the parsers names, whith no Base classes."""
name = os.path.splitext(os.path.basename(__file__))[0]
list_cls_names = (pyclbr.readmodule(name).keys())
list_no_base_cls_names = [cls_name for cls_name in list_cls_names
if cls_name[:4] != "Base"]
return list_no_base_cls_names
def _get_test_labels():
test_labels = []
for module in [name for _, name, _ in pkgutil.iter_modules([os.path.join("cms","tests")])]:
clsmembers = pyclbr.readmodule("cms.tests.%s" % module)
for clsname, cls in clsmembers.items():
for method, _ in cls.methods.items():
if method.startswith('test_'):
test_labels.append('cms.%s.%s' % (clsname, method))
test_labels = sorted(test_labels)
return test_labels
def __suite_case_str_to_class(self, suite_list):
suite_class_instance_list = []
for suite_item in suite_list:
suite_module_name = self.case_package + suite_item["name"]
suite_class_list = pyclbr.readmodule(suite_module_name).keys()
if len(suite_class_list) != 1:
self.logger.error("%s模块中测试类个数不为1，该Suite类将会被跳过.",
suite_module_name)
continue
suite_module = __import__(suite_module_name,
fromlist=suite_class_list)
suite_class_instance = suite_module.__dict__[suite_class_list[0]]
if not issubclass(suite_class_instance, PerformanceTestSuite):
self.logger.error("%s Suite类不是 %s 子类，该Suite类将会被跳过.",
suite_class_instance.__name__,
PerformanceTestSuite.__name__)
continue
case_class_instance_list = []
for case_item in suite_item["case"]:
case_module_name = self.case_package + case_item
case_class_list = pyclbr.readmodule(case_module_name).keys()
if len(case_class_list) != 1:
self.logger.error("%s模块中测试类个数不为1，该Case类将会被跳过.",
case_module_name)
continue
case_module = __import__(case_module_name,
fromlist=case_class_list)
case_class_instance = case_module.__dict__[case_class_list[0]]
if not issubclass(case_class_instance, PerformanceTestCase):
self.logger.error("%s Case类不是 %s 子类，该Case类将会被跳过.",
case_class_instance.__name__,
PerformanceTestCase.__name__)
continue
case_class_instance_list.append(case_class_instance)
suite_class_instance_list.append({
"suite": suite_class_instance,
"case": case_class_instance_list
})
return suite_class_instance_list
def register_all_models(module=None,path=None):
""" This function registers all modules in with the django admin.
The module name should be a string, and defaults to 'models' and the path can be a string, list or tuple"""
if module is None:
module='models'
if path is None:
path=os.path.dirname(os.path.abspath(__file__))
classes = pyclbr.readmodule(module,[path])
elif type(path) is str:
classes = pyclbr.readmodule(module,[path])
else:
classes = pyclbr.readmodule(module,path)
for model in classes:
# now the dirty part, check that the models are classes that inherit from models.Model
# if this inhertance is not explicit in the class call it will not be registered
for superclass in classes[model].super:
if re.search('models.Model',superclass):
# this could be a from module import * above this loop
exec('from %s import %s'%(module,classes[model].name))
exec('admin.site.register(%s)'%classes[model].name)
def list_models(self, module_name='models'):
import pyclbr
module = module_name
m = pyclbr.readmodule(module)
print('Models in %s module:' % module_name)
counter = 1
for value in m.itervalues():
if value.module == module:
base = value.super[0]
if hasattr(base, 'name') and base.name == 'BaseModel':
print('[%s]: %s' % (counter, value.name))
counter += 1
def _isAPluginModule(modName, path):
# ==========================================================================
# CAUTION: UNSAFE AND UGLY CODE
# unfortunately this is needed to overpass the pyclbr cache and allow
# checking of changed modules =/
reload(pyclbr)
# ==========================================================================
superclassesInMod = reduce(set.union, (_getAllClassesNames(c.super) \
for c in pyclbr.readmodule(modName, path).values()))
# I know... this is not a very comprehensive and secure test because the
# class name is not a unique identifier... but for the scope of this
# application, it's completely OK
return Plugin.__name__ in superclassesInMod
def get_modules():
path = os.path.dirname(os.path.realpath(__file__))
module_path = os.path.dirname(os.path.realpath(__file__)) + '/algorithms'
modules = []
for filename in glob.glob(module_path + '/*/*.py'):
if '__init__' in filename or 'setup' in filename:
continue
else:
classes = []
for key in pyclbr.readmodule(path_leaf(filename).replace('.py',''),[filename.replace(path_leaf(filename),"").replace(".py","")]):
classes.append(key)
modules.append(Module(classes,filename))
return modules
def getAvailableModules(self):
availableModules = []
for file in os.listdir(os.path.dirname("modules/")):
if file.endswith(".py"):
cl = pyclbr.readmodule(file[:-3])
for k, v in cl.items():
name = v.name
base = v.super
if not isinstance(v.super, str):
if not isinstance(v.super[0], str):
base = v.super[0].name
if "BotModule" in base:
availableModules.append(name)
return availableModules
def main():
parser = argparse.ArgumentParser(description='scan python \
source to find classes and stand-alone functions ',
epilog='scan_module.py -f <python file> | <path>/*.py')
parser.add_argument('-f', '--file', action='store',
nargs='*', dest='input_file', default=[],
help='specify one or more scan files.If \
no value is specified for this argument,\
it will scan all modules in the current \
directory. This is an optional argument.')
results = parser.parse_args()
if len(results.input_file) == 0:
for name in glob.glob('%s/*.py' % os.path.curdir):
results.input_file.append(name)
for f in results.input_file:
fp = dirname(abspath(f))
sys.path.append(fp + os.path.sep)
_sep = '.'
_modules_list = list(map(lambda x: basename(x).split(_sep)[0],
results.input_file))
_modules_len = len(_modules_list)
for fi in _modules_list:
time_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
print("\033[2;31m{} Start scanning module: {}\033[0m".format(time_now, fi))
example_data = pyclbr.readmodule(fi)
print('example_data --> {}'.format(example_data))
if example_data:
for name, class_data in sorted(example_data.items(),
key=lambda x: x[1].lineno):
print('name: {}, class_data: {}'.format(name, class_data))
scan_m = ScanModule()
scan_m.show_classes(name, class_data)
else:
module = __import__('%s' % fi)
all_functions = inspect.getmembers(module, inspect.isfunction)
print('File: {0}'.format(module))
for func in all_functions:
print('\tMethod: {0}'.format(func[0]))
print()
print('Scanned {} modules.'.format(_modules_len))
def getPlugins(module):
"""
Get plugins from module.
"""
plugins = pyclbr.readmodule(module)
out = []
for className in plugins:
pluginName = re.sub(r"^plugin(.*)", r"\1", className)
if (pluginName != className):
pluginType = Atom(plugins[className].super[0].name.lower())
try:
i = import_module(module)
triggers = getattr(i, className).clsTriggers()
out.append((pluginName, pluginType, triggers))
except:
return (Atom("bad_triggers"), className)
return List(out)
def _ensure_index(package, logger, do_print):
prefix = package.__name__ + "."
for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
full_modname = prefix+modname
outer_module = __import__(full_modname, fromlist="dummy")
if not ispkg:
#print "Found submodule %s (is a package: %s)" % (modname, ispkg)
#print "inspected: "+str(classes)
classes = pyclbr.readmodule(full_modname)
module = getattr(package, modname)
for key,value in classes.items():
#print full_modname
if 'Document' in value.super or 'WhiskeyNode' in value.super:
cls = getattr(module, value.name)
try:
inst = cls()
for index in inst.ENSURE_INDEXES:
if isinstance(index, list) or index not in inst.ENSURE_UNIQUE_INDEXES:
dbug_msg = "ensuring index cls: %s collection: %s index: %s " % (full_modname, inst.COLLECTION_NAME, index)
if logger is not None:
logger(dbug_msg)
elif do_print:
print dbug_msg
inst.COLLECTION.ensure_index(index)
for index in inst.ENSURE_UNIQUE_INDEXES:
dbug_msg = "ensuring unique index cls: %s collection: %s index: %s " % (full_modname, inst.COLLECTION_NAME, index)
if logger is not None:
logger(dbug_msg)
elif do_print:
print dbug_msg
if index not in inst.ENSURE_INDEXES:
raise Exception('All indexes in ENSURE_UNIQUE_INDEXES should also be in ENSURE_INDEXES')
inst.COLLECTION.ensure_index(index, unique=True)
except Exception, e:
pass
dbug_msg = "Failed to import %s %s" % (full_modname, str(e))
if logger is not None:
logger(dbug_msg)
elif do_print:
print dbug_msg
else:
_ensure_index(outer_module, logger, do_print)
def import_submodules(package, recursive=True):
""" Import all submodules of a module, recursively, including subpackages
:param package: package (name or actual module)
:type package: str | module
:rtype: dict[str, types.ModuleType]
"""
if isinstance(package, str):
package = importlib.import_module(package)
results = {}
modules = {}
modules['Errors'] = []
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
full_name = package.__name__ + '.' + name
try:
moduleObj = importlib.import_module(full_name)
results[full_name] = moduleObj
#print full_name
#module_classList = pyclbr.readmodule(full_name).values()
##pyclbr.readmodule doesn't read any functions, but class with methods
module_info = pyclbr.readmodule(full_name)
for className, data in sorted(module_info.items(), key=lambda x:x[1].lineno):
full_pre_name = package.__name__ + '.' +os.path.splitext(os.path.basename(data.file))[0]
if not modules.has_key(full_pre_name):
modules[full_pre_name] = []
if className not in modules[full_pre_name]:
modules[full_pre_name].append(className)
if recursive and is_pkg:
results.update(import_submodules(full_name))
except ImportError as e:
msg = full_name+ ': '+str(e);
results["ERROR: "+full_name] = msg
modules['Errors'].append(msg)
#print msg
return results
