def _check_clipboard_backend(backend): if backend == 'windows': return sys.platform.startswith('win32') if backend == 'qt': return (ub.modname_to_modpath('PyQt5') or ub.modname_to_modpath('PyQt4')) elif backend == 'gtk': return ub.modname_to_modpath('gtk') else: return pyperclip._executable_exists(backend)
def main(): import xdoctest import ubelt as ub import sys modpath = ub.modname_to_modpath('netharn.examples') name_to_path = {} submods = list(xdoctest.static_analysis.package_modpaths(modpath)) for submod in submods: modname = ub.augpath(submod, dpath='', ext='') if not modname.startswith('_'): name_to_path[modname] = submod print('name_to_path = {}'.format(ub.repr2(name_to_path, nl=1))) chosen = None for arg in sys.argv[1:2]: print('arg = {!r}'.format(arg)) if arg in name_to_path: chosen = name_to_path[arg] break print('chosen = {!r}'.format(chosen)) assert chosen is not None module = ub.import_module_from_path(chosen) print('module = {!r}'.format(module)) module.main()
def _exhaust(varname, modname, modpath): closer.debug( 'REWRITE ACCESSOR varname={!r}, modname={}, modpath={}'.format( varname, modname, modpath)) # Modify the current node definitions and recompute code # TODO: make more robust rewriter = RewriteModuleAccess(varname) for d_ in closer.body_defs.values(): rewriter.visit(d_.node) d_._code = unparse(d_.node) closer.debug('rewriter.accessed_attrs = {!r}'.format( rewriter.accessed_attrs)) # For each modified attribute, copy in the appropriate source. for subname in rewriter.accessed_attrs: submodname = modname + '.' + subname submodpath = ub.modname_to_modpath(submodname) if submodpath is not None: # if the accessor is to another module, exhaust until # we reach a non-module closer.debug('EXAUSTING: {}, {}, {}'.format( subname, submodname, submodpath)) _exhaust(subname, submodname, submodpath) else: # Otherwise we can directly add the referenced attribute closer.debug('FINALIZE: {} from {}'.format( subname, modpath)) closer.add_static(subname, modpath)
def expand_module(path): # TODO: use ubelt util_import instead import ubelt as ub _debug = 0 if _debug: import sys print('sys.base_exec_prefix = {!r}'.format(sys.base_exec_prefix)) print('sys.base_prefix = {!r}'.format(sys.base_prefix)) print('sys.exec_prefix = {!r}'.format(sys.exec_prefix)) print('sys.executable = {!r}'.format(sys.executable)) print('sys.implementation = {!r}'.format(sys.implementation)) print('sys.prefix = {!r}'.format(sys.prefix)) print('sys.version = {!r}'.format(sys.version)) print('sys.path = {!r}'.format(sys.path)) import sys extra_path = CONFIG.get('vimtk_sys_path') sys_path = sys.path + [ub.expandpath(p) for p in extra_path] print('expand path = {!r}'.format(path)) modparts = path.split('.') for i in reversed(range(1, len(modparts) + 1)): candidate = '.'.join(modparts[0:i]) print('candidate = {!r}'.format(candidate)) path = ub.modname_to_modpath(candidate, sys_path=sys_path) if path is not None: break print('expanded modname-to-path = {!r}'.format(path)) return path
def _import_from_definition(visitor, node): """ Ignore: from liberator.closer import * visitor = ImportVisitor.parse(module=module) print('visitor.definitions = {}'.format(ub.repr2(visitor.definitions, sv=1))) """ if node.level: # Handle relative imports if visitor.modpath is not None: try: rel_modpath = ub.split_modpath(abspath(visitor.modpath))[1] except ValueError: warnings.warn('modpath={} does not exist'.format( visitor.modpath)) rel_modpath = basename(abspath(visitor.modpath)) modparts = rel_modpath.replace('\\', '/').split('/') parts = modparts[:-node.level] prefix = '.'.join(parts) if node.module: prefix = prefix + '.' else: warnings.warn('Unable to rectify absolute import') prefix = '.' * node.level else: prefix = '' if node.module is not None: abs_modname = prefix + node.module else: abs_modname = prefix for alias in node.names: varname = alias.asname or alias.name if alias.asname: line = 'from {} import {} as {}'.format( abs_modname, alias.name, alias.asname) else: line = 'from {} import {}'.format(abs_modname, alias.name) absname = abs_modname + '.' + alias.name if varname == '*': # HACK abs_modpath = ub.modname_to_modpath(abs_modname) for d in ImportVisitor.parse( modpath=abs_modpath).definitions.values(): if not d.name.startswith('_'): yield d else: yield Definition(varname, node, code=line, absname=absname, modpath=visitor.modpath, modname=visitor.modname, native_modname=abs_modname, type='ImportFrom')
def in_pythonpath(modname): try: flag = check_module_installed(modname) except Exception: flag = False if flag: modpath = ub.modname_to_modpath(modname) if modpath is None: flag = False return flag
def _static_modname_to_modpath(modname, **kwargs): # Calls ub.modname_to_modpath with checks had = modname in sys.modules try: modpath = ub.modname_to_modpath(modname, **kwargs) except ValueError: modpath = None if not had: assert modname not in sys.modules, ( '{} should not be imported'.format(modname)) return modpath
def expand_module_prefix(path): # TODO: we could parse the AST to figure out if the prefix is an alias # for a known module. # Check if the path certainly looks like it could be a chain of python # attribute accessors. if re.match(r'^[\w\d_.]*$', path): extra_path = CONFIG.get('vimtk_sys_path') sys_path = sys.path + [ub.expandpath(p) for p in extra_path] parts = path.split('.') for i in reversed(range(len(parts))): prefix = '.'.join(parts[:i]) path = ub.modname_to_modpath(prefix, sys_path=sys_path) if path is not None: print('expanded prefix = {!r}'.format(path)) return path print('expanded prefix = {!r}'.format(None)) return None
def expand_module_attributes(closer, d): """ Args: d (Definition): the definition to expand """ # current_sourcecode = closer.current_sourcecode() # closed_visitor = ImportVisitor.parse(source=current_sourcecode) assert 'Import' in d.type varname = d.name varmodpath = ub.modname_to_modpath(d.absname) modname = d.absname def _exhaust(varname, modname, modpath): closer.debug( 'REWRITE ACCESSOR varname={!r}, modname={}, modpath={}'.format( varname, modname, modpath)) # Modify the current node definitions and recompute code # TODO: make more robust rewriter = RewriteModuleAccess(varname) for d_ in closer.body_defs.values(): rewriter.visit(d_.node) d_._code = unparse(d_.node) closer.debug('rewriter.accessed_attrs = {!r}'.format( rewriter.accessed_attrs)) # For each modified attribute, copy in the appropriate source. for subname in rewriter.accessed_attrs: submodname = modname + '.' + subname submodpath = ub.modname_to_modpath(submodname) if submodpath is not None: # if the accessor is to another module, exhaust until # we reach a non-module closer.debug('EXAUSTING: {}, {}, {}'.format( subname, submodname, submodpath)) _exhaust(subname, submodname, submodpath) else: # Otherwise we can directly add the referenced attribute closer.debug('FINALIZE: {} from {}'.format( subname, modpath)) closer.add_static(subname, modpath) _exhaust(varname, modname, varmodpath) d._code = '# ' + d.code
def modpath_coerce(modpath_coercable): """ if modpath_coercable is a name, statically converts it to a path Args: modpath_coercable (str | PathLike | ModuleType): something we can extract a path to a module from. Returns: str : the coerced modpath Example: >>> # xdoctest: +SKIP >>> from xdev.cli.docstr_stubgen import * # NOQA >>> import xdev >>> modpath_coercable = xdev >>> modpath = modpath_coerce(modpath_coercable) >>> print(f'modpath={modpath}') >>> assert modpath_coerce(modpath) == modpath >>> assert modpath_coerce(xdev.__name__) == modpath """ import ubelt as ub import types from os.path import exists import pathlib if isinstance(modpath_coercable, types.ModuleType): modpath = modpath_coercable.__file__ elif isinstance(modpath_coercable, pathlib.Path): modpath = modpath_coercable elif isinstance(modpath_coercable, str): modpath = ub.modname_to_modpath(modpath_coercable) if modpath is None: if exists(modpath_coercable): modpath = modpath_coercable else: raise ValueError( 'Cannot find module={}'.format(modpath_coercable)) else: raise TypeError('{}'.format(type(modpath_coercable))) modpath = ub.util_import.normalize_modpath(modpath) return modpath
def expand_module_attributes(closer, d): # current_sourcecode = closer.current_sourcecode() # closed_visitor = ImportVisitor.parse(source=current_sourcecode) assert 'Import' in d.type varname = d.name varmodpath = ub.modname_to_modpath(d.absname) # varmodpath = ub.modname_to_modpath(d.native_modname) expansion = d.absname # print('d = {!r}'.format(d)) def _exhaust(name, modname, modpath): if DEBUG: print('REWRITE ACCESSOR name={!r}, modname={}, modpath={}'. format(name, modname, modpath)) # Modify the current node definitions and recompute code # TODO: make more robust rewriter = RewriteModuleAccess(name) for d_ in closer.body_defs.values(): rewriter.visit(d_.node) d_._code = astunparse.unparse(d_.node) # print('rewriter.accessed_attrs = {!r}'.format(rewriter.accessed_attrs)) for subname in rewriter.accessed_attrs: submodname = modname + '.' + subname submodpath = ub.modname_to_modpath(submodname) if submodpath is not None: # if the accessor is to another module, exhaust until # we reach a non-module if DEBUG: print('EXAUSTING: {}, {}, {}'.format( subname, submodname, submodpath)) _exhaust(subname, submodname, submodpath) else: # Otherwise we can directly add the referenced attribute if DEBUG: print('FINALIZE: {} from {}'.format(subname, modpath)) closer.add_static(subname, modpath) _exhaust(varname, expansion, varmodpath) d._code = '# ' + d.code
import pytest # Prefer testing the installed version, but fallback to testing the # development version. try: import ubelt as ub except ImportError: print('running this test script requires ubelt') raise # Statically check if ``line_profiler`` is installed outside of the repo. # To do this, we make a copy of PYTHONPATH, remove the repodir, and use # ubelt to check to see if ``line_profiler`` can be resolved to a path. temp_path = list(map(abspath, sys.path)) if repo_dir in temp_path: temp_path.remove(repo_dir) modpath = ub.modname_to_modpath('line_profiler', sys_path=temp_path) if modpath is not None: # If it does, then import it. This should cause the installed version # to be used on further imports even if the repo_dir is in the path. print('Using installed version of line_profiler') module = ub.import_module_from_path(modpath, index=0) print('Installed module = {!r}'.format(module)) else: print('No installed version of line_profiler found') try: print('Changing dirs to test_dir={!r}'.format(test_dir)) os.chdir(test_dir) package_name = 'line_profiler' pytest_args = [
# Prefer testing the installed version, but fallback to testing the # development version. try: import ubelt as ub except ImportError: print('running this test script requires ubelt') raise package_name = 'PYPKG' # Statically check if ``package_name`` is installed outside of the repo. # To do this, we make a copy of PYTHONPATH, remove the repodir, and use # ubelt to check to see if ``package_name`` can be resolved to a path. temp_path = list(map(abspath, sys.path)) if repo_dir in temp_path: temp_path.remove(repo_dir) modpath = ub.modname_to_modpath(package_name, sys_path=temp_path) if modpath is not None: # If it does, then import it. This should cause the installed version # to be used on further imports even if the repo_dir is in the path. print(f'Using installed version of {package_name}') module = ub.import_module_from_path(modpath, index=0) print('Installed module = {!r}'.format(module)) else: print(f'No installed version of {package_name} found') try: print('Changing dirs to test_dir={!r}'.format(test_dir)) os.chdir(test_dir) pytest_args = [ '--cov-config', '../pyproject.toml',
def expand(closer, expand_names): """ Experimental feature. Remove all references to specific modules by directly copying in the referenced source code. If the code is referenced from a module, then the references will need to change as well. TODO: - [ ] Add special unique (mangled) suffixes to all expanded names to avoid name conflicts. Args: expand_name (List[str]): list of module names. For each module we expand any reference to that module in the closed source code by directly copying the referenced code into that file. This doesn't work in all cases, but it usually does. Reasons why this wouldn't work include trying to expand import from C-extension modules and expanding modules with complicated global-level logic. Ignore: >>> # Test a heavier duty class >>> from liberator.closer import * >>> import netharn as nh >>> obj = nh.device.MountedModel >>> #obj = nh.layers.ConvNormNd >>> obj = nh.data.CocoDataset >>> #expand_names = ['ubelt', 'progiter'] >>> closer = Closer() >>> closer.add_dynamic(obj) >>> closer.expand(expand_names) >>> #print('header_defs = ' + ub.repr2(closer.header_defs, si=1)) >>> #print('body_defs = ' + ub.repr2(closer.body_defs, si=1)) >>> print('SOURCE:') >>> text = closer.current_sourcecode() >>> print(text) """ closer.debug("!!! EXPANDING") # Expand references to internal modules flag = True while flag: # Associate all top-level modules with any possible expand_name # that might trigger them to be expanded. Note this does not # account for nested imports. expandable_definitions = ub.ddict(list) for d in closer.header_defs.values(): parts = d.native_modname.split('.') for i in range(1, len(parts) + 1): root = '.'.join(parts[:i]) expandable_definitions[root].append(d) closer.debug('expandable_definitions = {!r}'.format( list(expandable_definitions.keys()))) flag = False # current_sourcecode = closer.current_sourcecode() # closed_visitor = ImportVisitor.parse(source=current_sourcecode) for root in expand_names: needs_expansion = expandable_definitions.get(root, []) closer.debug('root = {!r}'.format(root)) closer.debug('needs_expansion = {!r}'.format(needs_expansion)) for d in needs_expansion: if d._expanded: continue flag = True # if d.absname == d.native_modname: if ub.modname_to_modpath(d.absname): closer.debug( 'TODO: NEED TO CLOSE module = {}'.format(d)) # import warnings # warnings.warn('Closing module {} may not be implemented'.format(d)) # definition is a module, need to expand its attributes closer.expand_module_attributes(d) d._expanded = True else: closer.debug( 'TODO: NEED TO CLOSE attribute varname = {}'. format(d)) import warnings # warnings.warn('Closing attribute {} may not be implemented'.format(d)) # definition is a non-module, directly copy in its code # We can directly replace this import statement by # copy-pasting the relevant code from the other module # (ASSUMING THERE ARE NO NAME CONFLICTS) assert d.type == 'ImportFrom' try: native_modpath = ub.modname_to_modpath( d.native_modname) if native_modpath is None: raise Exception( 'Cannot find the module path for modname={!r}. ' 'Are you missing an __init__.py?'.format( d.native_modname)) sub_closer = Closer(closer.tag + '.sub') sub_closer.add_static(d.name, native_modpath) # sub_visitor = sub_closer.visitors[d.native_modname] sub_closer.expand(expand_names) # sub_closer.close(sub_visitor) except NotAPythonFile as ex: warnings.warn( 'CANNOT EXPAND d = {!r}, REASON: {}'.format( d, repr(ex))) d._expanded = True raise continue except Exception as ex: warnings.warn( 'CANNOT EXPAND d = {!r}, REASON: {}'.format( d, repr(ex))) d._expanded = True raise continue # raise # Hack: remove the imported definition and add the explicit definition # TODO: FIXME: more robust modification and replacement d._code = '# ' + d.code d._expanded = True for d_ in sub_closer.header_defs.values(): closer._add_definition(d_) for d_ in sub_closer.body_defs.values(): closer._add_definition(d_) # print('sub_visitor = {!r}'.format(sub_visitor)) # closer.close(sub_visitor) closer.debug('CLOSED attribute d = {}'.format(d))
def autompl(verbose=0, recheck=False, force=None): """ Uses platform heuristics to automatically set the matplotlib backend. If no display is available it will be set to `agg`, otherwise we will try to use the cross-platform `Qt5Agg` backend. Args: verbose (int, default=0): verbosity level recheck (bool, default=False): if False, this function will not run if it has already been called (this can save a significant amount of time). force (str, default=None): backend to force to or "auto" Checks: export QT_DEBUG_PLUGINS=1 xdoctest -m kwplot.auto_backends autompl --check KWPLOT_UNSAFE=1 xdoctest -m kwplot.auto_backends autompl --check KWPLOT_UNSAFE=0 xdoctest -m kwplot.auto_backends autompl --check Example: >>> # xdoctest +REQUIRES(--check) >>> plt = autoplt(verbose=1) >>> plt.figure() References: https://stackoverflow.com/questions/637005/check-if-x-server-is-running """ global _AUTOMPL_WAS_RUN if force == 'auto': recheck = True force = None elif force is not None: set_mpl_backend(force) _AUTOMPL_WAS_RUN = True if recheck or not _AUTOMPL_WAS_RUN: if verbose: print('AUTOMPL') if sys.platform.startswith('win32'): # TODO: something reasonable pass else: DISPLAY = os.environ.get('DISPLAY', '') if DISPLAY: # Check if we can actually connect to X # NOTE: this call takes a significant amount of time info = ub.cmd('xdpyinfo', shell=True) if verbose: print('xdpyinfo-info = {}'.format(ub.repr2(info))) if info['ret'] != 0: DISPLAY = None if verbose: print(' * DISPLAY = {!r}'.format(DISPLAY)) if not DISPLAY: backend = 'agg' else: """ Note: May encounter error that crashes the program, not sure why this happens yet. The current workaround is to uninstall PyQt5, but that isn't sustainable. QObject::moveToThread: Current thread (0x7fe8d965d030) is not the object's thread (0x7fffb0f64340). Cannot move to target thread (0x7fe8d965d030) qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in "" even though it was found. This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem. Available platform plugins are: eglfs, linuxfb, minimal, minimalegl, offscreen, vnc, wayland-egl, wayland, wayland-xcomposite-egl, wayland-xcomposite-glx, webgl, xcb. UPDATE 2021-01-04: By setting export QT_DEBUG_PLUGINS=1 I was able to look at more debug information. It turns out that it was grabbing the xcb plugin from the opencv-python package. I uninstalled that package and then installed opencv-python-headless which does not include an xcb binary. However, now the it is missing "libxcb-xinerama". May be able to do something with: conda install -c conda-forge xorg-libxinerama # But that didnt work I had to pip uninstall PyQt5 # This seems to work correctly conda install -c anaconda pyqt """ if ub.modname_to_modpath('PyQt5'): try: import PyQt5 # NOQA from PyQt5 import QtCore # NOQA except ImportError: backend = 'agg' else: backend = 'Qt5Agg' KWPLOT_UNSAFE = os.environ.get('KWPLOT_UNSAFE', '') TRY_AVOID_CRASH = KWPLOT_UNSAFE.lower() not in [ '1', 'true', 'yes' ] if TRY_AVOID_CRASH and ub.LINUX: # HOLD UP. Lets try to avoid a crash. if 'cv2' in sys.modules: from os.path import dirname, join, exists cv2 = sys.modules['cv2'] cv2_mod_dpath = dirname(cv2.__file__) cv2_lib_dpath = join(cv2_mod_dpath, 'qt/plugins/platforms') cv2_qxcb_fpath = join(cv2_lib_dpath, 'libqxcb.so') qt_mod_dpath = dirname(QtCore.__file__) qt_lib_dpath = join(qt_mod_dpath, 'Qt/plugins/platforms') qt_qxcb_fpath = join(qt_lib_dpath, 'libqxcb.so') if exists(cv2_qxcb_fpath) and exists( qt_qxcb_fpath): # Can we use ldd to make the test better? import warnings warnings.warn( ub.paragraph(''' Autompl has detected libqxcb in PyQt and cv2. Falling back to agg to avoid a potential crash. This can be worked around by installing opencv-python-headless instead of opencv-python. Disable this check by setting the environ KWPLOT_UNSAFE=1 ''')) backend = 'agg' elif ub.modname_to_modpath('PyQt4'): try: import Qt4Agg # NOQA from PyQt4 import QtCore # NOQA except ImportError: backend = 'agg' else: backend = 'Qt4Agg' else: backend = 'agg' set_mpl_backend(backend, verbose=verbose) if 0: # TODO: # IF IN A NOTEBOOK, BE SURE TO SET INLINE BEHAVIOR # THIS EFFECTIVELY REPRODUCES THE %matplotlib inline behavior # BUT USING AN ACTUAL PYTHON FUNCTION shell = _current_ipython_session() if shell: shell.enable_matplotlib('inline') _AUTOMPL_WAS_RUN = True