def _static_parse_imports(modpath, imports=None, use_all=True): # from ubelt.meta import static_analysis as static # TODO: port some of this functionality over from xdoctest import static_analysis as static modname = static.modpath_to_modname(modpath) if imports is not None: import_paths = { m: static.modname_to_modpath(modname + '.' + m, hide_init=False) for m in imports } else: imports = [] import_paths = {} for sub_modpath in static.package_modpaths(modpath, with_pkg=True, recursive=False): # print('sub_modpath = {!r}'.format(sub_modpath)) sub_modname = static.modpath_to_modname(sub_modpath) rel_modname = sub_modname[len(modname) + 1:] if rel_modname.startswith('_'): continue if not rel_modname: continue import_paths[rel_modname] = sub_modpath imports.append(rel_modname) imports = sorted(imports) from_imports = [] for rel_modname in imports: sub_modpath = import_paths[rel_modname] with open(sub_modpath, 'r') as file: source = file.read() valid_callnames = None if use_all: try: valid_callnames = static.parse_static_value('__all__', source) except NameError: pass if valid_callnames is None: # The __all__ variable is not specified or we dont care top_level = static.TopLevelVisitor.parse(source) attrnames = list(top_level.assignments) + list( top_level.calldefs.keys()) # list of names we wont export by default invalid_callnames = dir(builtins) valid_callnames = [] for attr in attrnames: if '.' in attr or attr.startswith('_'): continue if attr in invalid_callnames: continue valid_callnames.append(attr) from_imports.append((rel_modname, sorted(valid_callnames))) return modname, imports, from_imports
def modpath_to_modname(modpath, hide_init=True, hide_main=False): """ Determines importable name from file path Converts the path to a module (__file__) to the importable python name (__name__) without importing the module. The filename is converted to a module name, and parent directories are recursively included until a directory without an __init__.py file is encountered. Args: modpath (str): module filepath hide_init (bool): removes the __init__ suffix (default True) hide_main (bool): removes the __main__ suffix (default False) Returns: str: modname Example: >>> import ubelt.util_import >>> modpath = ubelt.util_import.__file__ >>> print(modpath_to_modname(modpath)) ubelt.util_import """ from xdoctest import static_analysis as static return static.modpath_to_modname(modpath, hide_init, hide_main)
def _custom_import_modpath(modpath): import xdoctest.static_analysis as static dpath, rel_modpath = static.split_modpath(modpath) modname = static.modpath_to_modname(modpath) with PythonPathContext(dpath, index=0): module = import_module_from_name(modname) return module
def __init__(self, docsrc, modpath=None, callname=None, num=0, lineno=1, fpath=None, block_type=None, mode='pytest'): # if we know the google block type it is recorded self.block_type = block_type self.config = Config() self.modpath = modpath self.fpath = fpath if modpath is None: self.modname = '<modname?>' self.modpath = '<modpath?>' else: if fpath is not None: assert fpath == modpath, ( 'only specify fpath for non-python files') self.fpath = modpath self.modname = static.modpath_to_modname(modpath) if callname is None: self.callname = '<callname?>' else: self.callname = callname self.docsrc = docsrc self.lineno = lineno self.num = num self._parts = None self.failed_tb_lineno = None self.exc_info = None self.failed_part = None self.warn_list = None self.logged_evals = OrderedDict() self.logged_stdout = OrderedDict() self._unmatched_stdout = [] self._skipped_parts = [] self._runstate = None self.module = None # Maintain global variables that this test will have access to self.global_namespace = {} # Hint at what is running this doctest self.mode = mode
def _pkgutil_import_modpath(modpath): # nocover import six import xdoctest.static_analysis as static dpath, rel_modpath = static.split_modpath(modpath) modname = static.modpath_to_modname(modpath) if six.PY2: # nocover import imp module = imp.load_source(modname, modpath) elif sys.version_info[0:2] <= (3, 4): # nocover assert sys.version_info[0:2] <= (3, 2), '3.0 to 3.2 is not supported' from importlib.machinery import SourceFileLoader module = SourceFileLoader(modname, modpath).load_module() else: import importlib.util spec = importlib.util.spec_from_file_location(modname, modpath) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module
def import_module_from_path(modpath): """ Args: modpath (str): path to the module References: https://stackoverflow.com/questions/67631/import-module-given-path Example: >>> from xdoctest import utils >>> modpath = utils.__file__ >>> module = import_module_from_path(modpath) >>> assert module is utils """ # the importlib version doesnt work in pytest import xdoctest.static_analysis as static dpath, rel_modpath = static.split_modpath(modpath) modname = static.modpath_to_modname(modpath) with PythonPathContext(dpath): try: module = import_module_from_name(modname) except Exception: print('Failed to import modname={} with modpath={}'.format( modname, modpath)) raise # TODO: use this implementation once pytest fixes importlib # if six.PY2: # nocover # import imp # module = imp.load_source(modname, modpath) # elif sys.version_info[0:2] <= (3, 4): # nocover # assert sys.version_info[0:2] <= (3, 2), '3.0 to 3.2 is not supported' # from importlib.machinery import SourceFileLoader # module = SourceFileLoader(modname, modpath).load_module() # else: # import importlib.util # spec = importlib.util.spec_from_file_location(modname, modpath) # module = importlib.util.module_from_spec(spec) # spec.loader.exec_module(module) return module
def test_modpath_to_modname(): """ CommandLine: pytest testing/test_static.py::test_modpath_to_modname -s python testing/test_static.py test_modpath_to_modname """ with utils.TempDir() as temp: dpath = temp.dpath # Create a dummy package heirachy root = utils.ensuredir((dpath, '_tmproot')) sub1 = utils.ensuredir((root, 'sub1')) sub2 = utils.ensuredir((sub1, 'sub2')) root_init = touch((root, '__init__.py')) sub1_init = touch((sub1, '__init__.py')) sub2_init = touch((sub2, '__init__.py')) mod0 = touch((root, 'mod0.py')) mod1 = touch((sub1, 'mod1.py')) mod2 = touch((sub2, 'mod2.py')) root_main = touch((root, '__main__.py')) sub2_main = touch((sub2, '__main__.py')) bad1 = utils.ensuredir((root, 'bad1')) bad2 = utils.ensuredir((sub1, 'bad2')) b0 = touch((bad1, 'b0.py')) b1 = touch((bad2, 'b1.py')) with utils.PythonPathContext(dpath): assert static.modpath_to_modname(root) == '_tmproot' assert static.modpath_to_modname(sub1) == '_tmproot.sub1' assert static.modpath_to_modname(sub2) == '_tmproot.sub1.sub2' assert static.modpath_to_modname(mod0) == '_tmproot.mod0' assert static.modpath_to_modname(mod1) == '_tmproot.sub1.mod1' assert static.modpath_to_modname(mod2) == '_tmproot.sub1.sub2.mod2' assert static.modpath_to_modname(root_init) == '_tmproot' assert static.modpath_to_modname(sub1_init) == '_tmproot.sub1' assert static.modpath_to_modname(sub2_init) == '_tmproot.sub1.sub2' assert static.modpath_to_modname( root_init, hide_init=False) == '_tmproot.__init__' assert static.modpath_to_modname( sub1_init, hide_init=False) == '_tmproot.sub1.__init__' assert static.modpath_to_modname( sub2_init, hide_init=False) == '_tmproot.sub1.sub2.__init__' assert static.modpath_to_modname( root, hide_main=True, hide_init=False) == '_tmproot.__init__' assert static.modpath_to_modname( sub1, hide_main=True, hide_init=False) == '_tmproot.sub1.__init__' assert static.modpath_to_modname( sub2, hide_main=True, hide_init=False) == '_tmproot.sub1.sub2.__init__' assert static.modpath_to_modname( root, hide_main=False, hide_init=False) == '_tmproot.__init__' assert static.modpath_to_modname( sub1, hide_main=False, hide_init=False) == '_tmproot.sub1.__init__' assert static.modpath_to_modname( sub2, hide_main=False, hide_init=False) == '_tmproot.sub1.sub2.__init__' assert static.modpath_to_modname(root, hide_main=False, hide_init=True) == '_tmproot' assert static.modpath_to_modname(sub1, hide_main=False, hide_init=True) == '_tmproot.sub1' assert static.modpath_to_modname( sub2, hide_main=False, hide_init=True) == '_tmproot.sub1.sub2' assert static.modpath_to_modname( root_main, hide_main=False, hide_init=True) == '_tmproot.__main__' assert static.modpath_to_modname( sub2_main, hide_main=False, hide_init=True) == '_tmproot.sub1.sub2.__main__' assert static.modpath_to_modname( root_main, hide_main=False, hide_init=True) == '_tmproot.__main__' assert static.modpath_to_modname( sub2_main, hide_main=False, hide_init=True) == '_tmproot.sub1.sub2.__main__' assert static.modpath_to_modname(root_main, hide_main=True, hide_init=True) == '_tmproot' assert static.modpath_to_modname( sub2_main, hide_main=True, hide_init=True) == '_tmproot.sub1.sub2' assert static.modpath_to_modname(root_main, hide_main=True, hide_init=False) == '_tmproot' assert static.modpath_to_modname( sub2_main, hide_main=True, hide_init=False) == '_tmproot.sub1.sub2' # Non-existant / invalid modules should always be None for a, b in it.product([True, False], [True, False]): with pytest.raises(ValueError): static.modpath_to_modname(join(sub1, '__main__.py'), hide_main=a, hide_init=b) assert static.modpath_to_modname(b0, hide_main=a, hide_init=b) == 'b0' assert static.modpath_to_modname(b1, hide_main=a, hide_init=b) == 'b1' with pytest.raises(ValueError): static.modpath_to_modname(bad1, hide_main=a, hide_init=b) with pytest.raises(ValueError): static.modpath_to_modname(bad2, hide_main=a, hide_init=b) assert '_tmproot' not in sys.modules assert '_tmproot.mod0' not in sys.modules assert '_tmproot.sub1' not in sys.modules assert '_tmproot.sub1.mod1' not in sys.modules assert '_tmproot.sub1.sub2' not in sys.modules assert '_tmproot.sub1.mod2.mod2' not in sys.modules
def is_defined_by_module(item, module): """ Check if item is directly defined by a module. This check may not always work, especially for decorated functions. CommandLine: xdoctest -m xdoctest.dynamic_analysis is_defined_by_module Example: >>> from xdoctest import dynamic_analysis >>> item = dynamic_analysis.is_defined_by_module >>> module = dynamic_analysis >>> assert is_defined_by_module(item, module) >>> item = dynamic_analysis.six >>> assert not is_defined_by_module(item, module) >>> item = dynamic_analysis.six.print_ >>> assert not is_defined_by_module(item, module) >>> assert not is_defined_by_module(print, module) >>> # xdoctest: +REQUIRES(CPython) >>> import _ctypes >>> item = _ctypes.Array >>> module = _ctypes >>> assert is_defined_by_module(item, module) >>> item = _ctypes.CFuncPtr.restype >>> module = _ctypes >>> assert is_defined_by_module(item, module) """ from xdoctest import static_analysis as static target_modname = module.__name__ # invalid_types = (int, float, list, tuple, set) # if isinstance(item, invalid_types) or isinstance(item, six.string_type): # raise TypeError('can only test definitions for classes and functions') flag = False if isinstance(item, types.ModuleType): if not hasattr(item, '__file__'): try: # hack for cv2 and xfeatures2d name = static.modpath_to_modname(module.__file__) flag = name in str(item) except Exception: flag = False else: item_modpath = os.path.realpath(os.path.dirname(item.__file__)) mod_fpath = module.__file__.replace('.pyc', '.py') if not mod_fpath.endswith('__init__.py'): flag = False else: modpath = os.path.realpath(os.path.dirname(mod_fpath)) modpath = modpath.replace('.pyc', '.py') flag = item_modpath.startswith(modpath) else: # unwrap static/class/property methods if isinstance(item, property): item = item.fget if isinstance(item, staticmethod): item = item.__func__ if isinstance(item, classmethod): item = item.__func__ if getattr(item, '__module__', None) == target_modname: flag = True elif hasattr(item, '__objclass__'): # should we just unwrap objclass? parent = item.__objclass__ if getattr(parent, '__module__', None) == target_modname: flag = True if not flag: try: item_modname = _func_globals(item)['__name__'] if item_modname == target_modname: flag = True except AttributeError: pass return flag
def __init__(self, docsrc, modpath=None, callname=None, num=0, lineno=1, fpath=None, block_type=None, mode='pytest'): import types # if we know the google block type it is recorded self.block_type = block_type self.config = DoctestConfig() self.module = None self.modpath = modpath self.fpath = fpath if modpath is None: self.modname = self.UNKNOWN_MODNAME self.modpath = self.UNKNOWN_MODPATH elif isinstance(modpath, types.ModuleType): self.fpath = modpath self.module = modpath self.modname = modpath.__name__ self.modpath = getattr(self.module, '__file__', self.UNKNOWN_MODPATH) else: if fpath is not None: if fpath != modpath: raise AssertionError( 'only specify fpath for non-python files') self.fpath = modpath self.modname = static.modpath_to_modname(modpath) if callname is None: self.callname = self.UNKNOWN_CALLNAME else: self.callname = callname self.docsrc = docsrc self.lineno = lineno self.num = num self._parts = None self.failed_tb_lineno = None self.exc_info = None self.failed_part = None self.warn_list = None self.logged_evals = OrderedDict() self.logged_stdout = OrderedDict() self._unmatched_stdout = [] self._skipped_parts = [] self._runstate = None # Maintain global variables that this test will have access to self.global_namespace = {} # Hint at what is running this doctest self.mode = mode
def package_calldefs(modpath_or_name, exclude=[], ignore_syntax_errors=True): """ Statically generates all callable definitions in a module or package Args: modpath_or_name (str): path to or name of the module to be tested exclude (list): glob-patterns of file names to exclude ignore_syntax_errors (bool): if False raise an error when syntax errors occur in a doctest (default True) Example: >>> modpath_or_name = 'xdoctest.core' >>> testables = list(package_calldefs(modpath_or_name)) >>> assert len(testables) == 1 >>> calldefs, modpath = testables[0] >>> assert static.modpath_to_modname(modpath) == modpath_or_name >>> assert 'package_calldefs' in calldefs """ pkgpath = _rectify_to_modpath(modpath_or_name) modpaths = static.package_modpaths(pkgpath, with_pkg=True, with_libs=True) modpaths = list(modpaths) for modpath in modpaths: modname = static.modpath_to_modname(modpath) if any(fnmatch(modname, pat) for pat in exclude): continue if not exists(modpath): warnings.warn('Module {} does not exist. ' 'Is it an old pyc file?'.format(modname)) continue FORCE_DYNAMIC = '--xdoc-force-dynamic' in sys.argv # if false just skip extension modules ALLOW_DYNAMIC = '--no-xdoc-dynamic' not in sys.argv if FORCE_DYNAMIC: # Force dynamic parsing for everything do_dynamic = True else: # Some modules can only be parsed dynamically needs_dynamic = modpath.endswith(static._platform_pylib_exts()) do_dynamic = needs_dynamic and ALLOW_DYNAMIC if do_dynamic: try: calldefs = dynamic.parse_dynamic_calldefs(modpath) except ImportError as ex: # Some modules are just c modules msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {}' msg = msg.format(modname, modpath, ex) warnings.warn(msg) # real code contained errors except Exception as ex: msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {}' msg = msg.format(modname, modpath, ex) warnings.warn(msg) # real code contained errors raise else: yield calldefs, modpath else: try: calldefs = static.parse_calldefs(fpath=modpath) except SyntaxError as ex: # Handle error due to the actual code containing errors msg = 'Cannot parse module={} at path={}.\nCaused by: {}' msg = msg.format(modname, modpath, ex) if ignore_syntax_errors: warnings.warn(msg) # real code contained errors continue else: raise SyntaxError(msg) else: yield calldefs, modpath
def package_calldefs(modpath_or_name, exclude=[], ignore_syntax_errors=True, analysis='static'): """ Statically generates all callable definitions in a module or package Args: modpath_or_name (str): path to or name of the module to be tested exclude (List[str]): glob-patterns of file names to exclude ignore_syntax_errors (bool, default=True): if False raise an error when syntax errors occur in a doctest analysis (str, default='static'): if 'static', only static analysis is used to parse call definitions. If 'auto', uses dynamic analysis for compiled python extensions, but static analysis elsewhere, if 'dynamic', then dynamic analysis is used to parse all calldefs. Example: >>> modpath_or_name = 'xdoctest.core' >>> testables = list(package_calldefs(modpath_or_name)) >>> assert len(testables) == 1 >>> calldefs, modpath = testables[0] >>> assert static_analysis.modpath_to_modname(modpath) == modpath_or_name >>> assert 'package_calldefs' in calldefs """ pkgpath = _rectify_to_modpath(modpath_or_name) modpaths = static_analysis.package_modpaths(pkgpath, with_pkg=True, with_libs=True) modpaths = list(modpaths) for modpath in modpaths: modname = static_analysis.modpath_to_modname(modpath) if any(fnmatch(modname, pat) for pat in exclude): continue if not exists(modpath): warnings.warn( 'Module {} does not exist. ' 'Is it an old pyc file?'.format(modname)) continue # backwards compatibility hacks if '--allow-xdoc-dynamic' in sys.argv: analysis = 'auto' if '--xdoc-force-dynamic' in sys.argv: analysis = 'dynamic' needs_dynamic = modpath.endswith( static_analysis._platform_pylib_exts()) if analysis == 'static': do_dynamic = False elif analysis == 'dynamic': do_dynamic = True elif analysis == 'auto': do_dynamic = needs_dynamic else: raise KeyError(analysis) if do_dynamic: try: calldefs = dynamic_analysis.parse_dynamic_calldefs(modpath) except (ImportError, RuntimeError) as ex: # Some modules are just c modules msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {!r} {}' msg = msg.format(modname, modpath, type(ex), ex) warnings.warn(msg) except Exception as ex: msg = 'Cannot dynamically parse module={} at path={}.\nCaused by: {!r} {}' msg = msg.format(modname, modpath, type(ex), ex) warnings.warn(msg) raise else: yield calldefs, modpath else: if needs_dynamic: # Some modules can only be parsed dynamically continue try: calldefs = static_analysis.parse_calldefs(fpath=modpath) except SyntaxError as ex: # Handle error due to the actual code containing errors msg = 'Cannot parse module={} at path={}.\nCaused by: {}' msg = msg.format(modname, modpath, ex) if ignore_syntax_errors: warnings.warn(msg) # real code or docstr contained errors continue else: raise SyntaxError(msg) else: yield calldefs, modpath
def make_default_module_maintest(modpath, test_code=None, argv=None, force_full=False): """ Args: modname (str): module name Returns: str: text source code CommandLine: python -m utool.util_autogen --test-make_default_module_maintest References: http://legacy.python.org/dev/peps/pep-0338/ Example: >>> import sys, ubelt as ub >>> sys.path.append(ub.truepath('~/local/vim/rc/')) >>> from pyvim_funcs import * >>> import pyvim_funcs >>> modpath = pyvim_funcs.__file__ >>> argv = None >>> text = make_default_module_maintest(modpath) >>> print(text) """ # if not use_modrun: # if ub.WIN32: # augpath = 'set PYTHONPATH=%PYTHONPATH%' + os.pathsep + moddir # else: # augpath = 'export PYTHONPATH=$PYTHONPATH' + os.pathsep + moddir # cmdline = augpath + '\n' + cmdline import ubelt as ub from xdoctest import static_analysis as static modname = static.modpath_to_modname(modpath) moddir, rel_modpath = static.split_modpath(modpath) if not force_full: info = ub.cmd('python -c "import sys; print(sys.path)"') default_path = eval(info['out'], {}) is_importable = static.is_modname_importable(modname, exclude=['.'], sys_path=default_path) if not force_full and is_importable: cmdline = 'python -m ' + modname else: if ub.WIN32: modpath = ub.compressuser(modpath, home='%HOME%') cmdline = 'python -B ' + modpath.replace('\\', '/') else: modpath = ub.compressuser(modpath, home='~') cmdline = 'python ' + modpath if test_code is None: test_code = ub.codeblock( r''' import xdoctest xdoctest.doctest_module(__file__) ''') if argv is None: argv = ['all'] if argv is None: argv = [] cmdline_ = ub.indent(cmdline + ' ' + ' '.join(argv), ' ' * 8).lstrip(' ') test_code = ub.indent(test_code, ' ' * 4).lstrip(' ') text = ub.codeblock( r''' if __name__ == '__main__': {rr}""" CommandLine: {cmdline_} """ {test_code} ''' ).format(cmdline_=cmdline_, test_code=test_code, rr='{r}') text = text.format(r='r' if '\\' in text else '') return text