def test_all_imports_pyx():
""" Tests: test_all_imports_pyx: for rebuild, syntax correctness and internal imports
"""
print('::: TEST: test_all_imports_pyx()')
remove_files = []
remove_dirs = []
all_modules_path = []
for root, dirnames, filenames in walk(ROOT_PACKAGE_PATH):
all_modules_path.extend(glob(root + '/*.pyx'))
for pyx_module_file_path in all_modules_path:
module_filename = path_basename(pyx_module_file_path)
module_filename_no_ext = path_splitext(module_filename)[0]
cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path = build_cython_extension(
pyx_module_file_path,
cython_force_rebuild=True
)
so_loader = ExtensionFileLoader(module_filename_no_ext, cython_extension_module_path)
so_loader.load_module(module_filename_no_ext)
# add for cleanup
remove_files.append(cython_module_c_file_path)
remove_dirs.append(cython_build_dir_path)
# Cleanup
try:
for file_ in remove_files:
if path_exists(file_):
os_remove(file_)
for dir_ in remove_dirs:
if path_exists(dir_):
rmtree(dir_)
except Exception as err:
raise Exception('test_all_imports_pyx', 'Could not cython_clean_up: Exception: <{}>'.format(err))
def _import_path(path, module_name=MODULE_NAME):
"""
Function which imports ``path`` and returns it as a module. This is
meant to import pyd files produced by :meth:`Distribution._build` in
a Python 2/3 agnostic fashion.
:param str path:
The path to the file to import
:keyword str module_name:
Optional name of the module being imported. By default
this will use ``_pywincffi`` if no value is provided.
:raises ResourceNotFoundError:
Raised if ``path`` does not exist.
"""
if not isfile(path):
raise ResourceNotFoundError("Module path %r does not exist" % path)
elif ExtensionFileLoader is not None:
loader = ExtensionFileLoader(module_name, path)
return loader.load_module(module_name)
elif imp is not None: # pragma: no cover
return imp.load_dynamic(module_name, path)
else: # pragma: no cover
raise NotImplementedError(
"Neither `imp` or `ExtensionFileLoader` were imported")
def _import_path(path, module_name=MODULE_NAME):
"""
Function which imports ``path`` and returns it as a module. This is
meant to import pyd files produced by :meth:`Distribution._build` in
a Python 2/3 agnostic fashion.
:param str path:
The path to the file to import
:keyword str module_name:
Optional name of the module being imported. By default
this will use ``_pywincffi`` if no value is provided.
:raises ResourceNotFoundError:
Raised if ``path`` does not exist.
"""
logger.debug("_import_path(%r, module_name=%r)", path, module_name)
if not isfile(path):
raise ResourceNotFoundError("Module path %r does not exist" % path)
elif ExtensionFileLoader is not None:
loader = ExtensionFileLoader(module_name, path)
return loader.load_module(module_name)
elif imp is not None: # pragma: no cover
return imp.load_dynamic(module_name, path)
else: # pragma: no cover
raise NotImplementedError(
"Neither `imp` or `ExtensionFileLoader` were imported")
def test_cython(tmp_path):
srcdir = os.path.join(os.path.dirname(__file__), '..')
shutil.copytree(srcdir, tmp_path / 'random')
# build the examples and "install" them into a temporary directory
env = os.environ.copy()
subprocess.check_call([sys.executable, 'setup.py', 'build', 'install',
'--prefix', str(tmp_path / 'installdir'),
'--single-version-externally-managed',
'--record', str(tmp_path/ 'tmp_install_log.txt'),
],
cwd=str(tmp_path / 'random' / '_examples' / 'cython'),
env=env)
# get the path to the so's
so1 = so2 = None
with open(tmp_path /'tmp_install_log.txt') as fid:
for line in fid:
if 'extending.' in line:
so1 = line.strip()
if 'extending_distributions' in line:
so2 = line.strip()
assert so1 is not None
assert so2 is not None
# import the so's without adding the directory to sys.path
from importlib.machinery import ExtensionFileLoader
extending = ExtensionFileLoader('extending', so1).load_module()
extending_distributions = ExtensionFileLoader('extending_distributions', so2).load_module()
# actually test the cython c-extension
from numpy.random import PCG64
values = extending_distributions.uniforms_ex(PCG64(0), 10, 'd')
assert values.shape == (10,)
assert values.dtype == np.float64
def import_module(self, module_dir=None, silent=True):
"""Build an import the module
Keyword Args:
module_dir(str): The location to store all the files of the module (source, temporary objects,
shared object). Default to a temporary location.
silent(bool): Silent compilation. Default True
Returns:
The loaded C extension
Raises:
ImportError: if the C++ code could not be compiled or the module could not be loaded
"""
# Build module
cpp_code = self.get_cpp_code()
extension_kwargs = dict()
if self._enable_numpy:
import numpy as np
extension_kwargs['include_dirs'] = [np.get_include()]
module_filename = build_install_module(cpp_code, self._name, extension_kwargs=extension_kwargs,
module_dir=module_dir, silent=silent)
if module_filename is None:
raise ImportError('Module %s could not be load' % self._name)
# Load module
file_loader = ExtensionFileLoader(self._name, module_filename)
imported_module = file_loader.load_module(self._name)
return imported_module
def import_extension_module(module_name: str, path_to_extension_module: str):
# you don't want to know how long it took me to get this right
loader = ExtensionFileLoader(module_name, path_to_extension_module)
spec = spec_from_loader(module_name, loader)
module = module_from_spec(spec)
loader.exec_module(module)
return module
def test_cython(tmp_path):
srcdir = os.path.join(os.path.dirname(__file__), "..")
shutil.copytree(srcdir, tmp_path / "random")
# build the examples and "install" them into a temporary directory
build_dir = tmp_path / "random" / "_examples" / "cython"
subprocess.check_call(
[
sys.executable,
"setup.py",
"build",
"install",
"--prefix",
str(tmp_path / "installdir"),
"--single-version-externally-managed",
"--record",
str(tmp_path / "tmp_install_log.txt"),
],
cwd=str(build_dir),
)
# gh-16162: make sure numpy's __init__.pxd was used for cython
# not really part of this test, but it is a convenient place to check
with open(build_dir / "extending.c") as fid:
txt_to_find = 'NumPy API declarations from "numpy/__init__.pxd"'
for i, line in enumerate(fid):
if txt_to_find in line:
break
else:
assert False, "Could not find '{}' in C file, " "wrong pxd used".format(
txt_to_find)
# get the path to the so's
so1 = so2 = None
with open(tmp_path / "tmp_install_log.txt") as fid:
for line in fid:
if "extending." in line:
so1 = line.strip()
if "extending_distributions" in line:
so2 = line.strip()
assert so1 is not None
assert so2 is not None
# import the so's without adding the directory to sys.path
from importlib.machinery import ExtensionFileLoader
extending = ExtensionFileLoader("extending", so1).load_module()
extending_distributions = ExtensionFileLoader("extending_distributions",
so2).load_module()
# actually test the cython c-extension
from numpy.random import PCG64
values = extending_distributions.uniforms_ex(PCG64(0), 10, "d")
assert values.shape == (10, )
assert values.dtype == np.float64
def test_cython(tmp_path):
srcdir = os.path.join(os.path.dirname(__file__), '..')
shutil.copytree(srcdir, tmp_path / 'random')
# build the examples and "install" them into a temporary directory
build_dir = tmp_path / 'random' / '_examples' / 'cython'
subprocess.check_call(
[
sys.executable,
'setup.py',
'build',
'install',
'--prefix',
str(tmp_path / 'installdir'),
'--single-version-externally-managed',
'--record',
str(tmp_path / 'tmp_install_log.txt'),
],
cwd=str(build_dir),
)
# gh-16162: make sure numpy's __init__.pxd was used for cython
# not really part of this test, but it is a convenient place to check
with open(build_dir / 'extending.c') as fid:
txt_to_find = re.compile(
r'NumPy API declarations from "numpy\/__init__(\.([^.]*?))?.pxd"')
for i, line in enumerate(fid):
if txt_to_find.search(line):
break
else:
assert False, ("Could not find '{}' in C file, "
"wrong pxd used".format(txt_to_find))
# get the path to the so's
so1 = so2 = None
with open(tmp_path / 'tmp_install_log.txt') as fid:
for line in fid:
if 'extending.' in line:
so1 = line.strip()
if 'extending_distributions' in line:
so2 = line.strip()
assert so1 is not None
assert so2 is not None
# import the so's without adding the directory to sys.path
from importlib.machinery import ExtensionFileLoader
extending = ExtensionFileLoader('extending', so1).load_module()
extending_distributions = ExtensionFileLoader('extending_distributions',
so2).load_module()
# actually test the cython c-extension
from numpy.random import PCG64
values = extending_distributions.uniforms_ex(PCG64(0), 10, 'd')
assert values.shape == (10, )
assert values.dtype == np.float64
def __bootstrap__():
global __bootstrap__, __loader__, __file__
import sys, pkg_resources
from importlib.machinery import ExtensionFileLoader
__file__ = pkg_resources.resource_filename(__name__, '_SHA384.pyd')
__loader__ = None; del __bootstrap__, __loader__
ExtensionFileLoader(__name__,__file__).load_module()
def find_spec(self, fullname, path=None, target=None):
if not fullname.startswith('calibre_extensions'):
return
parts = fullname.split('.')
if parts[0] != 'calibre_extensions':
return
if len(parts) > 2:
return
is_package = len(parts) == 1
extension_name = None if is_package else parts[1]
path = os.path.join(plugins_loc, '__init__.py')
if extension_name:
if extension_name not in self.calibre_extensions:
return
for suffix in EXTENSION_SUFFIXES:
path = os.path.join(plugins_loc, extension_name + suffix)
if os.path.exists(path):
break
else:
return
return ModuleSpec(fullname,
ExtensionFileLoader(fullname, path),
is_package=is_package,
origin=path)
return ModuleSpec(fullname,
ExtensionsPackageLoader(self.calibre_extensions),
is_package=is_package,
origin=path)
def __bootstrap__():
global __bootstrap__, __loader__, __file__
import sys, pkg_resources
from importlib.machinery import ExtensionFileLoader
__file__ = pkg_resources.resource_filename(__name__, '_MD2.cpython-38-darwin.so')
__loader__ = None; del __bootstrap__, __loader__
ExtensionFileLoader(__name__,__file__).load_module()
def __bootstrap__():
global __bootstrap__, __loader__, __file__
import sys, pkg_resources
from importlib.machinery import ExtensionFileLoader
__file__ = pkg_resources.resource_filename(__name__, 'touch.pypy36-pp73-win32.pyd')
__loader__ = None; del __bootstrap__, __loader__
ExtensionFileLoader(__name__,__file__).exec_module()
def import_library ():
global api
import platform
ext = platform.system() == 'Windows' and '.dll' or '.so'
dir = os.path.dirname(os.path.abspath(__file__))
lib = os.path.join(dir, 'promact_is' + ext)
try:
if sys.version_info >= (3, 4):
from importlib.machinery import ExtensionFileLoader
from importlib.util import spec_from_file_location
from importlib.util import module_from_spec
loader = ExtensionFileLoader('promact_is', lib)
spec = spec_from_file_location('promact_is', loader=loader)
api = module_from_spec(spec)
spec.loader.exec_module(api)
else:
import imp
api = imp.load_dynamic('promact_is', lib)
except:
_, err, _ = sys.exc_info()
msg = 'Error while importing promact_is%s:\n%s' % (ext, err)
sys.exit(msg)
def __bootstrap__():
global __bootstrap__, __loader__, __file__
import sys, pkg_resources
from importlib.machinery import ExtensionFileLoader
__file__ = pkg_resources.resource_filename(__name__, '_sentencepiece.cpython-37m-x86_64-linux-gnu.so')
__loader__ = None; del __bootstrap__, __loader__
ExtensionFileLoader(__name__,__file__).exec_module()
def import_extension(path, modname):
import glob
npath = glob.glob(os.path.join(path, modname + '.*'))
# Blacklist fixes gh-65.
# We filter out any files that can be created by compilers which are not our actual compiled file.
# We cannot more directly search for our files because of differing platforms.
blacklist = ['dSYM', 'c', 'pyx']
npath = [x for x in npath if x.split('.')[-1] not in blacklist]
if len(npath) == 1:
npath = npath[0]
else:
raise ImportError('Failed to import',
os.path.join(path, modname + '.*'), ' len(npath)=',
len(npath))
try:
# Python 3.5+
import importlib.util
spec = importlib.util.spec_from_file_location(modname, npath)
foo = importlib.util.module_from_spec(spec)
spec.loader.exec_module(foo)
except (AttributeError, ImportError):
try:
# Python 3.4
from importlib.machinery import ExtensionFileLoader
foo = ExtensionFileLoader(modname, npath).load_module()
except ImportError:
# Python 2.7
import imp
foo = imp.load_dynamic(modname, npath)
return foo
def __bootstrap__():
global __bootstrap__, __loader__, __file__
import sys, pkg_resources
from importlib.machinery import ExtensionFileLoader
__file__ = pkg_resources.resource_filename(
__name__, '_warp_ctc.cpython-38-x86_64-linux-gnu.so')
__loader__ = None
del __bootstrap__, __loader__
ExtensionFileLoader(__name__, __file__).load_module()
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
else:
raise ValueError(modinfo)
def find_spec(self, name, path, target=None):
if name in self.overrides:
fnroot = self.overrides[name]
for sfx in self.EXTENSION_SUFFIXES:
fn = fnroot + sfx
if os.path.isfile(fn):
spec = ModuleSpec(
name=name,
loader=ExtensionFileLoader(name, fn),
origin=fn,
is_package=False,
)
return spec
def load_dynamic(name, path):
"""Load and initialize a module implemented as a dynamically loadable
shared library and return its module object. If the module was already
initialized, it will be initialized again.
"""
# imp module is deprecated since Python 3.4
if (sys.version_info >= (3, 4)):
from importlib.machinery import ExtensionFileLoader
from importlib.util import spec_from_loader, module_from_spec
loader = ExtensionFileLoader(name, path)
spec = spec_from_loader(name, loader, origin=path)
module = module_from_spec(spec)
spec.loader.exec_module(module)
return module
else:
import imp
return imp.load_dynamic(name, path)
def find_spec(cls, fullname, path=None, target=None):
"""
This finder is only for extension modules found within packages that
are included in the zip file (instead of as files on disk);
extension modules cannot be found within zip files but are stored in
the lib subdirectory; if the extension module is found in a package,
however, its name has been altered so this finder is needed.
"""
if path is None:
return None
suffixes = EXTENSION_SUFFIXES
for entry in sys.path:
if ".zip" in entry:
continue
for ext in suffixes:
location = os.path.join(entry, fullname + ext)
if os.path.isfile(location):
loader = ExtensionFileLoader(fullname, location)
return ModuleSpec(fullname, loader, origin=location)
return None
import os
import sys
from importlib.machinery import ExtensionFileLoader
DIR = os.path.abspath(os.path.dirname(__file__))
python_version = str(sys.version_info.major) + str(sys.version_info.minor)
cymjd = ExtensionFileLoader(
"cymjd",
os.path.join(DIR, "cymjd.cpython-{}-x86_64-linux-gnu.so".format(
python_version))).load_module()
checkderiv = cymjd.checkderiv
MjDerivative = cymjd.MjDerivative
def load_dynamic_ext(name, path):
''' Load compiled shared object and return as python module. '''
loader = ExtensionFileLoader(name, path)
return loader.load_module()
def load_dynamic_ext(name, path):
''' Load compiled shared object and return as python module. '''
loader = ExtensionFileLoader(name, path)
return loader.load_module()
def unbulk_dyn_load(name):
foo = ExtensionFileLoader("pygame." + name, name + ".pyd").load_module()
sys.modules[name] = foo
return foo
def load_dynamic(name, module_path):
return ExtensionFileLoader(name, module_path).load_module()
def unbulk_dyn_load_package_name(module_name, package_name, extension_name):
foo = ExtensionFileLoader(package_name, extension_name).load_module()
sys.modules[module_name] = foo
return foo
