def _static_parse_imports(modpath, imports=None, use_all=True):
# from ubelt.meta import static_analysis as static
# TODO: port some of this functionality over
from xdoctest import static_analysis as static
modname = static.modpath_to_modname(modpath)
if imports is not None:
import_paths = {
m: static.modname_to_modpath(modname + '.' + m, hide_init=False)
for m in imports
}
else:
imports = []
import_paths = {}
for sub_modpath in static.package_modpaths(modpath,
with_pkg=True,
recursive=False):
# print('sub_modpath = {!r}'.format(sub_modpath))
sub_modname = static.modpath_to_modname(sub_modpath)
rel_modname = sub_modname[len(modname) + 1:]
if rel_modname.startswith('_'):
continue
if not rel_modname:
continue
import_paths[rel_modname] = sub_modpath
imports.append(rel_modname)
imports = sorted(imports)
from_imports = []
for rel_modname in imports:
sub_modpath = import_paths[rel_modname]
with open(sub_modpath, 'r') as file:
source = file.read()
valid_callnames = None
if use_all:
try:
valid_callnames = static.parse_static_value('__all__', source)
except NameError:
pass
if valid_callnames is None:
# The __all__ variable is not specified or we dont care
top_level = static.TopLevelVisitor.parse(source)
attrnames = list(top_level.assignments) + list(
top_level.calldefs.keys())
# list of names we wont export by default
invalid_callnames = dir(builtins)
valid_callnames = []
for attr in attrnames:
if '.' in attr or attr.startswith('_'):
continue
if attr in invalid_callnames:
continue
valid_callnames.append(attr)
from_imports.append((rel_modname, sorted(valid_callnames)))
return modname, imports, from_imports
def modpath_to_modname(modpath, hide_init=True, hide_main=False):
"""
Determines importable name from file path
Converts the path to a module (__file__) to the importable python name
(__name__) without importing the module.
The filename is converted to a module name, and parent directories are
recursively included until a directory without an __init__.py file is
encountered.
Args:
modpath (str): module filepath
hide_init (bool): removes the __init__ suffix (default True)
hide_main (bool): removes the __main__ suffix (default False)
Returns:
str: modname
Example:
>>> import ubelt.util_import
>>> modpath = ubelt.util_import.__file__
>>> print(modpath_to_modname(modpath))
ubelt.util_import
"""
from xdoctest import static_analysis as static
return static.modpath_to_modname(modpath, hide_init, hide_main)
def _custom_import_modpath(modpath):
import xdoctest.static_analysis as static
dpath, rel_modpath = static.split_modpath(modpath)
modname = static.modpath_to_modname(modpath)
with PythonPathContext(dpath, index=0):
module = import_module_from_name(modname)
return module
def __init__(self,
docsrc,
modpath=None,
callname=None,
num=0,
lineno=1,
fpath=None,
block_type=None,
mode='pytest'):
# if we know the google block type it is recorded
self.block_type = block_type
self.config = Config()
self.modpath = modpath
self.fpath = fpath
if modpath is None:
self.modname = '<modname?>'
self.modpath = '<modpath?>'
else:
if fpath is not None:
assert fpath == modpath, (
'only specify fpath for non-python files')
self.fpath = modpath
self.modname = static.modpath_to_modname(modpath)
if callname is None:
self.callname = '<callname?>'
else:
self.callname = callname
self.docsrc = docsrc
self.lineno = lineno
self.num = num
self._parts = None
self.failed_tb_lineno = None
self.exc_info = None
self.failed_part = None
self.warn_list = None
self.logged_evals = OrderedDict()
self.logged_stdout = OrderedDict()
self._unmatched_stdout = []
self._skipped_parts = []
self._runstate = None
self.module = None
# Maintain global variables that this test will have access to
self.global_namespace = {}
# Hint at what is running this doctest
self.mode = mode
def _pkgutil_import_modpath(modpath): # nocover
import six
import xdoctest.static_analysis as static
dpath, rel_modpath = static.split_modpath(modpath)
modname = static.modpath_to_modname(modpath)
if six.PY2: # nocover
import imp
module = imp.load_source(modname, modpath)
elif sys.version_info[0:2] <= (3, 4): # nocover
assert sys.version_info[0:2] <= (3, 2), '3.0 to 3.2 is not supported'
from importlib.machinery import SourceFileLoader
module = SourceFileLoader(modname, modpath).load_module()
else:
import importlib.util
spec = importlib.util.spec_from_file_location(modname, modpath)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def import_module_from_path(modpath):
"""
Args:
modpath (str): path to the module
References:
https://stackoverflow.com/questions/67631/import-module-given-path
Example:
>>> from xdoctest import utils
>>> modpath = utils.__file__
>>> module = import_module_from_path(modpath)
>>> assert module is utils
"""
# the importlib version doesnt work in pytest
import xdoctest.static_analysis as static
dpath, rel_modpath = static.split_modpath(modpath)
modname = static.modpath_to_modname(modpath)
with PythonPathContext(dpath):
try:
module = import_module_from_name(modname)
except Exception:
print('Failed to import modname={} with modpath={}'.format(
modname, modpath))
raise
# TODO: use this implementation once pytest fixes importlib
# if six.PY2: # nocover
# import imp
# module = imp.load_source(modname, modpath)
# elif sys.version_info[0:2] <= (3, 4): # nocover
# assert sys.version_info[0:2] <= (3, 2), '3.0 to 3.2 is not supported'
# from importlib.machinery import SourceFileLoader
# module = SourceFileLoader(modname, modpath).load_module()
# else:
# import importlib.util
# spec = importlib.util.spec_from_file_location(modname, modpath)
# module = importlib.util.module_from_spec(spec)
# spec.loader.exec_module(module)
return module
def test_modpath_to_modname():
"""
CommandLine:
pytest testing/test_static.py::test_modpath_to_modname -s
python testing/test_static.py test_modpath_to_modname
"""
with utils.TempDir() as temp:
dpath = temp.dpath
# Create a dummy package heirachy
root = utils.ensuredir((dpath, '_tmproot'))
sub1 = utils.ensuredir((root, 'sub1'))
sub2 = utils.ensuredir((sub1, 'sub2'))
root_init = touch((root, '__init__.py'))
sub1_init = touch((sub1, '__init__.py'))
sub2_init = touch((sub2, '__init__.py'))
mod0 = touch((root, 'mod0.py'))
mod1 = touch((sub1, 'mod1.py'))
mod2 = touch((sub2, 'mod2.py'))
root_main = touch((root, '__main__.py'))
sub2_main = touch((sub2, '__main__.py'))
bad1 = utils.ensuredir((root, 'bad1'))
bad2 = utils.ensuredir((sub1, 'bad2'))
b0 = touch((bad1, 'b0.py'))
b1 = touch((bad2, 'b1.py'))
with utils.PythonPathContext(dpath):
assert static.modpath_to_modname(root) == '_tmproot'
assert static.modpath_to_modname(sub1) == '_tmproot.sub1'
assert static.modpath_to_modname(sub2) == '_tmproot.sub1.sub2'
assert static.modpath_to_modname(mod0) == '_tmproot.mod0'
assert static.modpath_to_modname(mod1) == '_tmproot.sub1.mod1'
assert static.modpath_to_modname(mod2) == '_tmproot.sub1.sub2.mod2'
assert static.modpath_to_modname(root_init) == '_tmproot'
assert static.modpath_to_modname(sub1_init) == '_tmproot.sub1'
assert static.modpath_to_modname(sub2_init) == '_tmproot.sub1.sub2'
assert static.modpath_to_modname(
root_init, hide_init=False) == '_tmproot.__init__'
assert static.modpath_to_modname(
sub1_init, hide_init=False) == '_tmproot.sub1.__init__'
assert static.modpath_to_modname(
sub2_init, hide_init=False) == '_tmproot.sub1.sub2.__init__'
assert static.modpath_to_modname(
root, hide_main=True, hide_init=False) == '_tmproot.__init__'
assert static.modpath_to_modname(
sub1, hide_main=True,
hide_init=False) == '_tmproot.sub1.__init__'
assert static.modpath_to_modname(
sub2, hide_main=True,
hide_init=False) == '_tmproot.sub1.sub2.__init__'
assert static.modpath_to_modname(
root, hide_main=False, hide_init=False) == '_tmproot.__init__'
assert static.modpath_to_modname(
sub1, hide_main=False,
hide_init=False) == '_tmproot.sub1.__init__'
assert static.modpath_to_modname(
sub2, hide_main=False,
hide_init=False) == '_tmproot.sub1.sub2.__init__'
assert static.modpath_to_modname(root,
hide_main=False,
hide_init=True) == '_tmproot'
assert static.modpath_to_modname(sub1,
hide_main=False,
hide_init=True) == '_tmproot.sub1'
assert static.modpath_to_modname(
sub2, hide_main=False, hide_init=True) == '_tmproot.sub1.sub2'
assert static.modpath_to_modname(
root_main, hide_main=False,
hide_init=True) == '_tmproot.__main__'
assert static.modpath_to_modname(
sub2_main, hide_main=False,
hide_init=True) == '_tmproot.sub1.sub2.__main__'
assert static.modpath_to_modname(
root_main, hide_main=False,
hide_init=True) == '_tmproot.__main__'
assert static.modpath_to_modname(
sub2_main, hide_main=False,
hide_init=True) == '_tmproot.sub1.sub2.__main__'
assert static.modpath_to_modname(root_main,
hide_main=True,
hide_init=True) == '_tmproot'
assert static.modpath_to_modname(
sub2_main, hide_main=True,
hide_init=True) == '_tmproot.sub1.sub2'
assert static.modpath_to_modname(root_main,
hide_main=True,
hide_init=False) == '_tmproot'
assert static.modpath_to_modname(
sub2_main, hide_main=True,
hide_init=False) == '_tmproot.sub1.sub2'
# Non-existant / invalid modules should always be None
for a, b in it.product([True, False], [True, False]):
with pytest.raises(ValueError):
static.modpath_to_modname(join(sub1, '__main__.py'),
hide_main=a,
hide_init=b)
assert static.modpath_to_modname(b0, hide_main=a,
hide_init=b) == 'b0'
assert static.modpath_to_modname(b1, hide_main=a,
hide_init=b) == 'b1'
with pytest.raises(ValueError):
static.modpath_to_modname(bad1, hide_main=a, hide_init=b)
with pytest.raises(ValueError):
static.modpath_to_modname(bad2, hide_main=a, hide_init=b)
assert '_tmproot' not in sys.modules
assert '_tmproot.mod0' not in sys.modules
assert '_tmproot.sub1' not in sys.modules
assert '_tmproot.sub1.mod1' not in sys.modules
assert '_tmproot.sub1.sub2' not in sys.modules
assert '_tmproot.sub1.mod2.mod2' not in sys.modules
def is_defined_by_module(item, module):
"""
Check if item is directly defined by a module.
This check may not always work, especially for decorated functions.
CommandLine:
xdoctest -m xdoctest.dynamic_analysis is_defined_by_module
Example:
>>> from xdoctest import dynamic_analysis
>>> item = dynamic_analysis.is_defined_by_module
>>> module = dynamic_analysis
>>> assert is_defined_by_module(item, module)
>>> item = dynamic_analysis.six
>>> assert not is_defined_by_module(item, module)
>>> item = dynamic_analysis.six.print_
>>> assert not is_defined_by_module(item, module)
>>> assert not is_defined_by_module(print, module)
>>> # xdoctest: +REQUIRES(CPython)
>>> import _ctypes
>>> item = _ctypes.Array
>>> module = _ctypes
>>> assert is_defined_by_module(item, module)
>>> item = _ctypes.CFuncPtr.restype
>>> module = _ctypes
>>> assert is_defined_by_module(item, module)
"""
from xdoctest import static_analysis as static
target_modname = module.__name__
# invalid_types = (int, float, list, tuple, set)
# if isinstance(item, invalid_types) or isinstance(item, six.string_type):
# raise TypeError('can only test definitions for classes and functions')
flag = False
if isinstance(item, types.ModuleType):
if not hasattr(item, '__file__'):
try:
# hack for cv2 and xfeatures2d
name = static.modpath_to_modname(module.__file__)
flag = name in str(item)
except Exception:
flag = False
else:
item_modpath = os.path.realpath(os.path.dirname(item.__file__))
mod_fpath = module.__file__.replace('.pyc', '.py')
if not mod_fpath.endswith('__init__.py'):
flag = False
else:
modpath = os.path.realpath(os.path.dirname(mod_fpath))
modpath = modpath.replace('.pyc', '.py')
flag = item_modpath.startswith(modpath)
else:
# unwrap static/class/property methods
if isinstance(item, property):
item = item.fget
if isinstance(item, staticmethod):
item = item.__func__
if isinstance(item, classmethod):
item = item.__func__
if getattr(item, '__module__', None) == target_modname:
flag = True
elif hasattr(item, '__objclass__'):
# should we just unwrap objclass?
parent = item.__objclass__
if getattr(parent, '__module__', None) == target_modname:
flag = True
if not flag:
try:
item_modname = _func_globals(item)['__name__']
if item_modname == target_modname:
flag = True
except AttributeError:
pass
return flag
def __init__(self,
docsrc,
modpath=None,
callname=None,
num=0,
lineno=1,
fpath=None,
block_type=None,
mode='pytest'):
import types
# if we know the google block type it is recorded
self.block_type = block_type
self.config = DoctestConfig()
self.module = None
self.modpath = modpath
self.fpath = fpath
if modpath is None:
self.modname = self.UNKNOWN_MODNAME
self.modpath = self.UNKNOWN_MODPATH
elif isinstance(modpath, types.ModuleType):
self.fpath = modpath
self.module = modpath
self.modname = modpath.__name__
self.modpath = getattr(self.module, '__file__',
self.UNKNOWN_MODPATH)
else:
if fpath is not None:
if fpath != modpath:
raise AssertionError(
'only specify fpath for non-python files')
self.fpath = modpath
self.modname = static.modpath_to_modname(modpath)
if callname is None:
self.callname = self.UNKNOWN_CALLNAME
else:
self.callname = callname
self.docsrc = docsrc
self.lineno = lineno
self.num = num
self._parts = None
self.failed_tb_lineno = None
self.exc_info = None
self.failed_part = None
self.warn_list = None
self.logged_evals = OrderedDict()
self.logged_stdout = OrderedDict()
self._unmatched_stdout = []
self._skipped_parts = []
self._runstate = None
# Maintain global variables that this test will have access to
self.global_namespace = {}
# Hint at what is running this doctest
self.mode = mode
def package_calldefs(modpath_or_name, exclude=[], ignore_syntax_errors=True):
"""
Statically generates all callable definitions in a module or package
Args:
modpath_or_name (str): path to or name of the module to be tested
exclude (list): glob-patterns of file names to exclude
ignore_syntax_errors (bool): if False raise an error when syntax errors
occur in a doctest (default True)
Example:
>>> modpath_or_name = 'xdoctest.core'
>>> testables = list(package_calldefs(modpath_or_name))
>>> assert len(testables) == 1
>>> calldefs, modpath = testables[0]
>>> assert static.modpath_to_modname(modpath) == modpath_or_name
>>> assert 'package_calldefs' in calldefs
"""
pkgpath = _rectify_to_modpath(modpath_or_name)
modpaths = static.package_modpaths(pkgpath, with_pkg=True, with_libs=True)
modpaths = list(modpaths)
for modpath in modpaths:
modname = static.modpath_to_modname(modpath)
if any(fnmatch(modname, pat) for pat in exclude):
continue
if not exists(modpath):
warnings.warn('Module {} does not exist. '
'Is it an old pyc file?'.format(modname))
continue
FORCE_DYNAMIC = '--xdoc-force-dynamic' in sys.argv
# if false just skip extension modules
ALLOW_DYNAMIC = '--no-xdoc-dynamic' not in sys.argv
if FORCE_DYNAMIC:
# Force dynamic parsing for everything
do_dynamic = True
else:
# Some modules can only be parsed dynamically
needs_dynamic = modpath.endswith(static._platform_pylib_exts())
do_dynamic = needs_dynamic and ALLOW_DYNAMIC
if do_dynamic:
try:
calldefs = dynamic.parse_dynamic_calldefs(modpath)
except ImportError as ex:
# Some modules are just c modules
msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {}'
msg = msg.format(modname, modpath, ex)
warnings.warn(msg) # real code contained errors
except Exception as ex:
msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {}'
msg = msg.format(modname, modpath, ex)
warnings.warn(msg) # real code contained errors
raise
else:
yield calldefs, modpath
else:
try:
calldefs = static.parse_calldefs(fpath=modpath)
except SyntaxError as ex:
# Handle error due to the actual code containing errors
msg = 'Cannot parse module={} at path={}.\nCaused by: {}'
msg = msg.format(modname, modpath, ex)
if ignore_syntax_errors:
warnings.warn(msg) # real code contained errors
continue
else:
raise SyntaxError(msg)
else:
yield calldefs, modpath
def package_calldefs(modpath_or_name, exclude=[], ignore_syntax_errors=True,
analysis='static'):
"""
Statically generates all callable definitions in a module or package
Args:
modpath_or_name (str): path to or name of the module to be tested
exclude (List[str]): glob-patterns of file names to exclude
ignore_syntax_errors (bool, default=True):
if False raise an error when syntax errors occur in a doctest
analysis (str, default='static'):
if 'static', only static analysis is used to parse call
definitions. If 'auto', uses dynamic analysis for compiled python
extensions, but static analysis elsewhere, if 'dynamic', then
dynamic analysis is used to parse all calldefs.
Example:
>>> modpath_or_name = 'xdoctest.core'
>>> testables = list(package_calldefs(modpath_or_name))
>>> assert len(testables) == 1
>>> calldefs, modpath = testables[0]
>>> assert static_analysis.modpath_to_modname(modpath) == modpath_or_name
>>> assert 'package_calldefs' in calldefs
"""
pkgpath = _rectify_to_modpath(modpath_or_name)
modpaths = static_analysis.package_modpaths(pkgpath, with_pkg=True,
with_libs=True)
modpaths = list(modpaths)
for modpath in modpaths:
modname = static_analysis.modpath_to_modname(modpath)
if any(fnmatch(modname, pat) for pat in exclude):
continue
if not exists(modpath):
warnings.warn(
'Module {} does not exist. '
'Is it an old pyc file?'.format(modname))
continue
# backwards compatibility hacks
if '--allow-xdoc-dynamic' in sys.argv:
analysis = 'auto'
if '--xdoc-force-dynamic' in sys.argv:
analysis = 'dynamic'
needs_dynamic = modpath.endswith(
static_analysis._platform_pylib_exts())
if analysis == 'static':
do_dynamic = False
elif analysis == 'dynamic':
do_dynamic = True
elif analysis == 'auto':
do_dynamic = needs_dynamic
else:
raise KeyError(analysis)
if do_dynamic:
try:
calldefs = dynamic_analysis.parse_dynamic_calldefs(modpath)
except (ImportError, RuntimeError) as ex:
# Some modules are just c modules
msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {!r} {}'
msg = msg.format(modname, modpath, type(ex), ex)
warnings.warn(msg)
except Exception as ex:
msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {!r} {}'
msg = msg.format(modname, modpath, type(ex), ex)
warnings.warn(msg)
raise
else:
yield calldefs, modpath
else:
if needs_dynamic:
# Some modules can only be parsed dynamically
continue
try:
calldefs = static_analysis.parse_calldefs(fpath=modpath)
except SyntaxError as ex:
# Handle error due to the actual code containing errors
msg = 'Cannot parse module={} at path={}.\nCaused by: {}'
msg = msg.format(modname, modpath, ex)
if ignore_syntax_errors:
warnings.warn(msg) # real code or docstr contained errors
continue
else:
raise SyntaxError(msg)
else:
yield calldefs, modpath
def make_default_module_maintest(modpath, test_code=None, argv=None,
force_full=False):
"""
Args:
modname (str): module name
Returns:
str: text source code
CommandLine:
python -m utool.util_autogen --test-make_default_module_maintest
References:
http://legacy.python.org/dev/peps/pep-0338/
Example:
>>> import sys, ubelt as ub
>>> sys.path.append(ub.truepath('~/local/vim/rc/'))
>>> from pyvim_funcs import *
>>> import pyvim_funcs
>>> modpath = pyvim_funcs.__file__
>>> argv = None
>>> text = make_default_module_maintest(modpath)
>>> print(text)
"""
# if not use_modrun:
# if ub.WIN32:
# augpath = 'set PYTHONPATH=%PYTHONPATH%' + os.pathsep + moddir
# else:
# augpath = 'export PYTHONPATH=$PYTHONPATH' + os.pathsep + moddir
# cmdline = augpath + '\n' + cmdline
import ubelt as ub
from xdoctest import static_analysis as static
modname = static.modpath_to_modname(modpath)
moddir, rel_modpath = static.split_modpath(modpath)
if not force_full:
info = ub.cmd('python -c "import sys; print(sys.path)"')
default_path = eval(info['out'], {})
is_importable = static.is_modname_importable(modname, exclude=['.'],
sys_path=default_path)
if not force_full and is_importable:
cmdline = 'python -m ' + modname
else:
if ub.WIN32:
modpath = ub.compressuser(modpath, home='%HOME%')
cmdline = 'python -B ' + modpath.replace('\\', '/')
else:
modpath = ub.compressuser(modpath, home='~')
cmdline = 'python ' + modpath
if test_code is None:
test_code = ub.codeblock(
r'''
import xdoctest
xdoctest.doctest_module(__file__)
''')
if argv is None:
argv = ['all']
if argv is None:
argv = []
cmdline_ = ub.indent(cmdline + ' ' + ' '.join(argv), ' ' * 8).lstrip(' ')
test_code = ub.indent(test_code, ' ' * 4).lstrip(' ')
text = ub.codeblock(
r'''
if __name__ == '__main__':
{rr}"""
CommandLine:
{cmdline_}
"""
{test_code}
'''
).format(cmdline_=cmdline_, test_code=test_code, rr='{r}')
text = text.format(r='r' if '\\' in text else '')
return text
