def _check_clipboard_backend(backend):
if backend == 'windows':
return sys.platform.startswith('win32')
if backend == 'qt':
return (ub.modname_to_modpath('PyQt5')
or ub.modname_to_modpath('PyQt4'))
elif backend == 'gtk':
return ub.modname_to_modpath('gtk')
else:
return pyperclip._executable_exists(backend)
def main():
import xdoctest
import ubelt as ub
import sys
modpath = ub.modname_to_modpath('netharn.examples')
name_to_path = {}
submods = list(xdoctest.static_analysis.package_modpaths(modpath))
for submod in submods:
modname = ub.augpath(submod, dpath='', ext='')
if not modname.startswith('_'):
name_to_path[modname] = submod
print('name_to_path = {}'.format(ub.repr2(name_to_path, nl=1)))
chosen = None
for arg in sys.argv[1:2]:
print('arg = {!r}'.format(arg))
if arg in name_to_path:
chosen = name_to_path[arg]
break
print('chosen = {!r}'.format(chosen))
assert chosen is not None
module = ub.import_module_from_path(chosen)
print('module = {!r}'.format(module))
module.main()
def _exhaust(varname, modname, modpath):
closer.debug(
'REWRITE ACCESSOR varname={!r}, modname={}, modpath={}'.format(
varname, modname, modpath))
# Modify the current node definitions and recompute code
# TODO: make more robust
rewriter = RewriteModuleAccess(varname)
for d_ in closer.body_defs.values():
rewriter.visit(d_.node)
d_._code = unparse(d_.node)
closer.debug('rewriter.accessed_attrs = {!r}'.format(
rewriter.accessed_attrs))
# For each modified attribute, copy in the appropriate source.
for subname in rewriter.accessed_attrs:
submodname = modname + '.' + subname
submodpath = ub.modname_to_modpath(submodname)
if submodpath is not None:
# if the accessor is to another module, exhaust until
# we reach a non-module
closer.debug('EXAUSTING: {}, {}, {}'.format(
subname, submodname, submodpath))
_exhaust(subname, submodname, submodpath)
else:
# Otherwise we can directly add the referenced attribute
closer.debug('FINALIZE: {} from {}'.format(
subname, modpath))
closer.add_static(subname, modpath)
def expand_module(path):
# TODO: use ubelt util_import instead
import ubelt as ub
_debug = 0
if _debug:
import sys
print('sys.base_exec_prefix = {!r}'.format(sys.base_exec_prefix))
print('sys.base_prefix = {!r}'.format(sys.base_prefix))
print('sys.exec_prefix = {!r}'.format(sys.exec_prefix))
print('sys.executable = {!r}'.format(sys.executable))
print('sys.implementation = {!r}'.format(sys.implementation))
print('sys.prefix = {!r}'.format(sys.prefix))
print('sys.version = {!r}'.format(sys.version))
print('sys.path = {!r}'.format(sys.path))
import sys
extra_path = CONFIG.get('vimtk_sys_path')
sys_path = sys.path + [ub.expandpath(p) for p in extra_path]
print('expand path = {!r}'.format(path))
modparts = path.split('.')
for i in reversed(range(1, len(modparts) + 1)):
candidate = '.'.join(modparts[0:i])
print('candidate = {!r}'.format(candidate))
path = ub.modname_to_modpath(candidate, sys_path=sys_path)
if path is not None:
break
print('expanded modname-to-path = {!r}'.format(path))
return path
def _import_from_definition(visitor, node):
"""
Ignore:
from liberator.closer import *
visitor = ImportVisitor.parse(module=module)
print('visitor.definitions = {}'.format(ub.repr2(visitor.definitions, sv=1)))
"""
if node.level:
# Handle relative imports
if visitor.modpath is not None:
try:
rel_modpath = ub.split_modpath(abspath(visitor.modpath))[1]
except ValueError:
warnings.warn('modpath={} does not exist'.format(
visitor.modpath))
rel_modpath = basename(abspath(visitor.modpath))
modparts = rel_modpath.replace('\\', '/').split('/')
parts = modparts[:-node.level]
prefix = '.'.join(parts)
if node.module:
prefix = prefix + '.'
else:
warnings.warn('Unable to rectify absolute import')
prefix = '.' * node.level
else:
prefix = ''
if node.module is not None:
abs_modname = prefix + node.module
else:
abs_modname = prefix
for alias in node.names:
varname = alias.asname or alias.name
if alias.asname:
line = 'from {} import {} as {}'.format(
abs_modname, alias.name, alias.asname)
else:
line = 'from {} import {}'.format(abs_modname, alias.name)
absname = abs_modname + '.' + alias.name
if varname == '*':
# HACK
abs_modpath = ub.modname_to_modpath(abs_modname)
for d in ImportVisitor.parse(
modpath=abs_modpath).definitions.values():
if not d.name.startswith('_'):
yield d
else:
yield Definition(varname,
node,
code=line,
absname=absname,
modpath=visitor.modpath,
modname=visitor.modname,
native_modname=abs_modname,
type='ImportFrom')
def in_pythonpath(modname):
try:
flag = check_module_installed(modname)
except Exception:
flag = False
if flag:
modpath = ub.modname_to_modpath(modname)
if modpath is None:
flag = False
return flag
def _static_modname_to_modpath(modname, **kwargs):
# Calls ub.modname_to_modpath with checks
had = modname in sys.modules
try:
modpath = ub.modname_to_modpath(modname, **kwargs)
except ValueError:
modpath = None
if not had:
assert modname not in sys.modules, (
'{} should not be imported'.format(modname))
return modpath
def expand_module_prefix(path):
# TODO: we could parse the AST to figure out if the prefix is an alias
# for a known module.
# Check if the path certainly looks like it could be a chain of python
# attribute accessors.
if re.match(r'^[\w\d_.]*$', path):
extra_path = CONFIG.get('vimtk_sys_path')
sys_path = sys.path + [ub.expandpath(p) for p in extra_path]
parts = path.split('.')
for i in reversed(range(len(parts))):
prefix = '.'.join(parts[:i])
path = ub.modname_to_modpath(prefix, sys_path=sys_path)
if path is not None:
print('expanded prefix = {!r}'.format(path))
return path
print('expanded prefix = {!r}'.format(None))
return None
def expand_module_attributes(closer, d):
"""
Args:
d (Definition): the definition to expand
"""
# current_sourcecode = closer.current_sourcecode()
# closed_visitor = ImportVisitor.parse(source=current_sourcecode)
assert 'Import' in d.type
varname = d.name
varmodpath = ub.modname_to_modpath(d.absname)
modname = d.absname
def _exhaust(varname, modname, modpath):
closer.debug(
'REWRITE ACCESSOR varname={!r}, modname={}, modpath={}'.format(
varname, modname, modpath))
# Modify the current node definitions and recompute code
# TODO: make more robust
rewriter = RewriteModuleAccess(varname)
for d_ in closer.body_defs.values():
rewriter.visit(d_.node)
d_._code = unparse(d_.node)
closer.debug('rewriter.accessed_attrs = {!r}'.format(
rewriter.accessed_attrs))
# For each modified attribute, copy in the appropriate source.
for subname in rewriter.accessed_attrs:
submodname = modname + '.' + subname
submodpath = ub.modname_to_modpath(submodname)
if submodpath is not None:
# if the accessor is to another module, exhaust until
# we reach a non-module
closer.debug('EXAUSTING: {}, {}, {}'.format(
subname, submodname, submodpath))
_exhaust(subname, submodname, submodpath)
else:
# Otherwise we can directly add the referenced attribute
closer.debug('FINALIZE: {} from {}'.format(
subname, modpath))
closer.add_static(subname, modpath)
_exhaust(varname, modname, varmodpath)
d._code = '# ' + d.code
def modpath_coerce(modpath_coercable):
"""
if modpath_coercable is a name, statically converts it to a path
Args:
modpath_coercable (str | PathLike | ModuleType):
something we can extract a path to a module from.
Returns:
str : the coerced modpath
Example:
>>> # xdoctest: +SKIP
>>> from xdev.cli.docstr_stubgen import * # NOQA
>>> import xdev
>>> modpath_coercable = xdev
>>> modpath = modpath_coerce(modpath_coercable)
>>> print(f'modpath={modpath}')
>>> assert modpath_coerce(modpath) == modpath
>>> assert modpath_coerce(xdev.__name__) == modpath
"""
import ubelt as ub
import types
from os.path import exists
import pathlib
if isinstance(modpath_coercable, types.ModuleType):
modpath = modpath_coercable.__file__
elif isinstance(modpath_coercable, pathlib.Path):
modpath = modpath_coercable
elif isinstance(modpath_coercable, str):
modpath = ub.modname_to_modpath(modpath_coercable)
if modpath is None:
if exists(modpath_coercable):
modpath = modpath_coercable
else:
raise ValueError(
'Cannot find module={}'.format(modpath_coercable))
else:
raise TypeError('{}'.format(type(modpath_coercable)))
modpath = ub.util_import.normalize_modpath(modpath)
return modpath
def expand_module_attributes(closer, d):
# current_sourcecode = closer.current_sourcecode()
# closed_visitor = ImportVisitor.parse(source=current_sourcecode)
assert 'Import' in d.type
varname = d.name
varmodpath = ub.modname_to_modpath(d.absname)
# varmodpath = ub.modname_to_modpath(d.native_modname)
expansion = d.absname
# print('d = {!r}'.format(d))
def _exhaust(name, modname, modpath):
if DEBUG:
print('REWRITE ACCESSOR name={!r}, modname={}, modpath={}'.
format(name, modname, modpath))
# Modify the current node definitions and recompute code
# TODO: make more robust
rewriter = RewriteModuleAccess(name)
for d_ in closer.body_defs.values():
rewriter.visit(d_.node)
d_._code = astunparse.unparse(d_.node)
# print('rewriter.accessed_attrs = {!r}'.format(rewriter.accessed_attrs))
for subname in rewriter.accessed_attrs:
submodname = modname + '.' + subname
submodpath = ub.modname_to_modpath(submodname)
if submodpath is not None:
# if the accessor is to another module, exhaust until
# we reach a non-module
if DEBUG:
print('EXAUSTING: {}, {}, {}'.format(
subname, submodname, submodpath))
_exhaust(subname, submodname, submodpath)
else:
# Otherwise we can directly add the referenced attribute
if DEBUG:
print('FINALIZE: {} from {}'.format(subname, modpath))
closer.add_static(subname, modpath)
_exhaust(varname, expansion, varmodpath)
d._code = '# ' + d.code
import pytest
# Prefer testing the installed version, but fallback to testing the
# development version.
try:
import ubelt as ub
except ImportError:
print('running this test script requires ubelt')
raise
# Statically check if ``line_profiler`` is installed outside of the repo.
# To do this, we make a copy of PYTHONPATH, remove the repodir, and use
# ubelt to check to see if ``line_profiler`` can be resolved to a path.
temp_path = list(map(abspath, sys.path))
if repo_dir in temp_path:
temp_path.remove(repo_dir)
modpath = ub.modname_to_modpath('line_profiler', sys_path=temp_path)
if modpath is not None:
# If it does, then import it. This should cause the installed version
# to be used on further imports even if the repo_dir is in the path.
print('Using installed version of line_profiler')
module = ub.import_module_from_path(modpath, index=0)
print('Installed module = {!r}'.format(module))
else:
print('No installed version of line_profiler found')
try:
print('Changing dirs to test_dir={!r}'.format(test_dir))
os.chdir(test_dir)
package_name = 'line_profiler'
pytest_args = [
# Prefer testing the installed version, but fallback to testing the
# development version.
try:
import ubelt as ub
except ImportError:
print('running this test script requires ubelt')
raise
package_name = 'PYPKG'
# Statically check if ``package_name`` is installed outside of the repo.
# To do this, we make a copy of PYTHONPATH, remove the repodir, and use
# ubelt to check to see if ``package_name`` can be resolved to a path.
temp_path = list(map(abspath, sys.path))
if repo_dir in temp_path:
temp_path.remove(repo_dir)
modpath = ub.modname_to_modpath(package_name, sys_path=temp_path)
if modpath is not None:
# If it does, then import it. This should cause the installed version
# to be used on further imports even if the repo_dir is in the path.
print(f'Using installed version of {package_name}')
module = ub.import_module_from_path(modpath, index=0)
print('Installed module = {!r}'.format(module))
else:
print(f'No installed version of {package_name} found')
try:
print('Changing dirs to test_dir={!r}'.format(test_dir))
os.chdir(test_dir)
pytest_args = [
'--cov-config', '../pyproject.toml',
def expand(closer, expand_names):
"""
Experimental feature. Remove all references to specific modules by
directly copying in the referenced source code. If the code is
referenced from a module, then the references will need to change as
well.
TODO:
- [ ] Add special unique (mangled) suffixes to all expanded names
to avoid name conflicts.
Args:
expand_name (List[str]): list of module names. For each module
we expand any reference to that module in the closed source
code by directly copying the referenced code into that file.
This doesn't work in all cases, but it usually does.
Reasons why this wouldn't work include trying to expand
import from C-extension modules and expanding modules with
complicated global-level logic.
Ignore:
>>> # Test a heavier duty class
>>> from liberator.closer import *
>>> import netharn as nh
>>> obj = nh.device.MountedModel
>>> #obj = nh.layers.ConvNormNd
>>> obj = nh.data.CocoDataset
>>> #expand_names = ['ubelt', 'progiter']
>>> closer = Closer()
>>> closer.add_dynamic(obj)
>>> closer.expand(expand_names)
>>> #print('header_defs = ' + ub.repr2(closer.header_defs, si=1))
>>> #print('body_defs = ' + ub.repr2(closer.body_defs, si=1))
>>> print('SOURCE:')
>>> text = closer.current_sourcecode()
>>> print(text)
"""
closer.debug("!!! EXPANDING")
# Expand references to internal modules
flag = True
while flag:
# Associate all top-level modules with any possible expand_name
# that might trigger them to be expanded. Note this does not
# account for nested imports.
expandable_definitions = ub.ddict(list)
for d in closer.header_defs.values():
parts = d.native_modname.split('.')
for i in range(1, len(parts) + 1):
root = '.'.join(parts[:i])
expandable_definitions[root].append(d)
closer.debug('expandable_definitions = {!r}'.format(
list(expandable_definitions.keys())))
flag = False
# current_sourcecode = closer.current_sourcecode()
# closed_visitor = ImportVisitor.parse(source=current_sourcecode)
for root in expand_names:
needs_expansion = expandable_definitions.get(root, [])
closer.debug('root = {!r}'.format(root))
closer.debug('needs_expansion = {!r}'.format(needs_expansion))
for d in needs_expansion:
if d._expanded:
continue
flag = True
# if d.absname == d.native_modname:
if ub.modname_to_modpath(d.absname):
closer.debug(
'TODO: NEED TO CLOSE module = {}'.format(d))
# import warnings
# warnings.warn('Closing module {} may not be implemented'.format(d))
# definition is a module, need to expand its attributes
closer.expand_module_attributes(d)
d._expanded = True
else:
closer.debug(
'TODO: NEED TO CLOSE attribute varname = {}'.
format(d))
import warnings
# warnings.warn('Closing attribute {} may not be implemented'.format(d))
# definition is a non-module, directly copy in its code
# We can directly replace this import statement by
# copy-pasting the relevant code from the other module
# (ASSUMING THERE ARE NO NAME CONFLICTS)
assert d.type == 'ImportFrom'
try:
native_modpath = ub.modname_to_modpath(
d.native_modname)
if native_modpath is None:
raise Exception(
'Cannot find the module path for modname={!r}. '
'Are you missing an __init__.py?'.format(
d.native_modname))
sub_closer = Closer(closer.tag + '.sub')
sub_closer.add_static(d.name, native_modpath)
# sub_visitor = sub_closer.visitors[d.native_modname]
sub_closer.expand(expand_names)
# sub_closer.close(sub_visitor)
except NotAPythonFile as ex:
warnings.warn(
'CANNOT EXPAND d = {!r}, REASON: {}'.format(
d, repr(ex)))
d._expanded = True
raise
continue
except Exception as ex:
warnings.warn(
'CANNOT EXPAND d = {!r}, REASON: {}'.format(
d, repr(ex)))
d._expanded = True
raise
continue
# raise
# Hack: remove the imported definition and add the explicit definition
# TODO: FIXME: more robust modification and replacement
d._code = '# ' + d.code
d._expanded = True
for d_ in sub_closer.header_defs.values():
closer._add_definition(d_)
for d_ in sub_closer.body_defs.values():
closer._add_definition(d_)
# print('sub_visitor = {!r}'.format(sub_visitor))
# closer.close(sub_visitor)
closer.debug('CLOSED attribute d = {}'.format(d))
def autompl(verbose=0, recheck=False, force=None):
"""
Uses platform heuristics to automatically set the matplotlib backend.
If no display is available it will be set to `agg`, otherwise we will try
to use the cross-platform `Qt5Agg` backend.
Args:
verbose (int, default=0): verbosity level
recheck (bool, default=False): if False, this function will not run if
it has already been called (this can save a significant amount of
time).
force (str, default=None): backend to force to or "auto"
Checks:
export QT_DEBUG_PLUGINS=1
xdoctest -m kwplot.auto_backends autompl --check
KWPLOT_UNSAFE=1 xdoctest -m kwplot.auto_backends autompl --check
KWPLOT_UNSAFE=0 xdoctest -m kwplot.auto_backends autompl --check
Example:
>>> # xdoctest +REQUIRES(--check)
>>> plt = autoplt(verbose=1)
>>> plt.figure()
References:
https://stackoverflow.com/questions/637005/check-if-x-server-is-running
"""
global _AUTOMPL_WAS_RUN
if force == 'auto':
recheck = True
force = None
elif force is not None:
set_mpl_backend(force)
_AUTOMPL_WAS_RUN = True
if recheck or not _AUTOMPL_WAS_RUN:
if verbose:
print('AUTOMPL')
if sys.platform.startswith('win32'):
# TODO: something reasonable
pass
else:
DISPLAY = os.environ.get('DISPLAY', '')
if DISPLAY:
# Check if we can actually connect to X
# NOTE: this call takes a significant amount of time
info = ub.cmd('xdpyinfo', shell=True)
if verbose:
print('xdpyinfo-info = {}'.format(ub.repr2(info)))
if info['ret'] != 0:
DISPLAY = None
if verbose:
print(' * DISPLAY = {!r}'.format(DISPLAY))
if not DISPLAY:
backend = 'agg'
else:
"""
Note:
May encounter error that crashes the program, not sure why
this happens yet. The current workaround is to uninstall
PyQt5, but that isn't sustainable.
QObject::moveToThread: Current thread (0x7fe8d965d030) is not the object's thread (0x7fffb0f64340).
Cannot move to target thread (0x7fe8d965d030)
qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in "" even though it was found.
This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem.
Available platform plugins are: eglfs, linuxfb, minimal, minimalegl, offscreen, vnc, wayland-egl, wayland, wayland-xcomposite-egl, wayland-xcomposite-glx, webgl, xcb.
UPDATE 2021-01-04:
By setting
export QT_DEBUG_PLUGINS=1
I was able to look at more debug information. It turns out
that it was grabbing the xcb plugin from the opencv-python
package. I uninstalled that package and then installed
opencv-python-headless which does not include an xcb
binary. However, now the it is missing "libxcb-xinerama".
May be able to do something with:
conda install -c conda-forge xorg-libxinerama
# But that didnt work I had to
pip uninstall PyQt5
# This seems to work correctly
conda install -c anaconda pyqt
"""
if ub.modname_to_modpath('PyQt5'):
try:
import PyQt5 # NOQA
from PyQt5 import QtCore # NOQA
except ImportError:
backend = 'agg'
else:
backend = 'Qt5Agg'
KWPLOT_UNSAFE = os.environ.get('KWPLOT_UNSAFE', '')
TRY_AVOID_CRASH = KWPLOT_UNSAFE.lower() not in [
'1', 'true', 'yes'
]
if TRY_AVOID_CRASH and ub.LINUX:
# HOLD UP. Lets try to avoid a crash.
if 'cv2' in sys.modules:
from os.path import dirname, join, exists
cv2 = sys.modules['cv2']
cv2_mod_dpath = dirname(cv2.__file__)
cv2_lib_dpath = join(cv2_mod_dpath,
'qt/plugins/platforms')
cv2_qxcb_fpath = join(cv2_lib_dpath,
'libqxcb.so')
qt_mod_dpath = dirname(QtCore.__file__)
qt_lib_dpath = join(qt_mod_dpath,
'Qt/plugins/platforms')
qt_qxcb_fpath = join(qt_lib_dpath,
'libqxcb.so')
if exists(cv2_qxcb_fpath) and exists(
qt_qxcb_fpath):
# Can we use ldd to make the test better?
import warnings
warnings.warn(
ub.paragraph('''
Autompl has detected libqxcb in PyQt
and cv2. Falling back to agg to avoid
a potential crash. This can be worked
around by installing
opencv-python-headless instead of
opencv-python.
Disable this check by setting the
environ KWPLOT_UNSAFE=1
'''))
backend = 'agg'
elif ub.modname_to_modpath('PyQt4'):
try:
import Qt4Agg # NOQA
from PyQt4 import QtCore # NOQA
except ImportError:
backend = 'agg'
else:
backend = 'Qt4Agg'
else:
backend = 'agg'
set_mpl_backend(backend, verbose=verbose)
if 0:
# TODO:
# IF IN A NOTEBOOK, BE SURE TO SET INLINE BEHAVIOR
# THIS EFFECTIVELY REPRODUCES THE %matplotlib inline behavior
# BUT USING AN ACTUAL PYTHON FUNCTION
shell = _current_ipython_session()
if shell:
shell.enable_matplotlib('inline')
_AUTOMPL_WAS_RUN = True
