def pyx_to_dll(filename, ext=None, force_rebuild=0):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename)
path, name = os.path.split(filename)
if not ext:
modname, extension = os.path.splitext(name)
assert extension == ".pyx", extension
ext = Extension(name=modname, sources=[filename])
if DEBUG:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
dist = Distribution({"script_name": None, "script_args": args})
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
dist.cmdclass = {'build_ext': build_ext}
build = dist.get_command_obj('build')
build.build_base = os.path.join(path, "_pyxbld")
try:
ok = dist.parse_command_line()
except DistutilsArgError, msg:
raise
def pyx_to_dll(filename, ext = None, force_rebuild = 0):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename)
path, name = os.path.split(filename)
if not ext:
modname, extension = os.path.splitext(name)
assert extension == ".pyx", extension
ext = Extension(name=modname, sources=[filename])
if DEBUG:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
dist = Distribution({"script_name": None, "script_args": args})
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
dist.cmdclass = {'build_ext': build_ext}
build = dist.get_command_obj('build')
build.build_base = os.path.join(path, "_pyxbld")
try:
ok = dist.parse_command_line()
except DistutilsArgError, msg:
raise
def build_cython_extension(py_or_pyx_file_path, cython_force_rebuild=True):
""" Build a cython extension from a `.py` or `.pyx` file
- build will be done in a sub-folder named `_pyxbld` in the py_or_pyx_file_path
:param py_or_pyx_file_path: (str) path to a `.py` or `.pyx` file
:param cython_force_rebuild: (bool) If True the cython extension is rebuild even if it was already build
:return: (tuple) cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
"""
module_dir = path_dirname(py_or_pyx_file_path)
module__cython_name = path_splitext(path_basename(py_or_pyx_file_path))[0]
cython_module_c_file_path = path_join(module_dir,
module__cython_name + '.c')
cython_build_dir_path = path_join(module_dir, '_pyxbld')
args = ['--quiet', 'build_ext', '--build-lib', module_dir]
if cython_force_rebuild:
args.append('--force')
dist = Distribution({'script_name': None, 'script_args': args})
dist.ext_modules = [
Extension(name=module__cython_name, sources=[py_or_pyx_file_path])
]
dist.cmdclass = {'build_ext': cython_build_ext}
build = dist.get_command_obj('build')
build.build_base = cython_build_dir_path
try:
dist.parse_command_line()
except DistutilsArgError as err:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' DistutilsArgError: <{}>'.format(err)
])
try:
obj_build_ext = dist.get_command_obj('build_ext')
dist.run_commands()
cython_extension_module_path = obj_build_ext.get_outputs()[0]
if path_dirname(py_or_pyx_file_path) != module_dir:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' <module_dir> differs from final <cython_module_dir>',
' module_dir: <{}>'.format(module_dir),
' cython_module_dir: <{}>'.format(
path_dirname(py_or_pyx_file_path))
])
except Exception as err:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' Exception: <{}>'.format(err)
])
return cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
def build_cython_extension(py_or_pyx_file_path, cython_force_rebuild=True):
""" Build a cython extension from a `.py` or `.pyx` file
- build will be done in a sub-folder named `_pyxbld` in the py_or_pyx_file_path
:param py_or_pyx_file_path: (str) path to a `.py` or `.pyx` file
:param cython_force_rebuild: (bool) If True the cython extension is rebuild even if it was already build
:return: (tuple) cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
"""
module_dir = path_dirname(py_or_pyx_file_path)
module__cython_name = path_splitext(path_basename(py_or_pyx_file_path))[0]
cython_module_c_file_path = path_join(module_dir, module__cython_name + '.c')
cython_build_dir_path = path_join(module_dir, '_pyxbld')
args = ['--quiet', 'build_ext', '--build-lib', module_dir]
if cython_force_rebuild:
args.append('--force')
dist = Distribution({'script_name': None, 'script_args': args})
dist.ext_modules = [Extension(name=module__cython_name, sources=[py_or_pyx_file_path])]
dist.cmdclass = {'build_ext': cython_build_ext}
build = dist.get_command_obj('build')
build.build_base = cython_build_dir_path
try:
dist.parse_command_line()
except DistutilsArgError as err:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' DistutilsArgError: <{}>'.format(err)
])
try:
obj_build_ext = dist.get_command_obj('build_ext')
dist.run_commands()
cython_extension_module_path = obj_build_ext.get_outputs()[0]
if path_dirname(py_or_pyx_file_path) != module_dir:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' <module_dir> differs from final <cython_module_dir>',
' module_dir: <{}>'.format(module_dir),
' cython_module_dir: <{}>'.format(path_dirname(py_or_pyx_file_path))
])
except Exception as err:
raise Err('utils.build_cython_extension', [
'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path),
' Exception: <{}>'.format(err)
])
return cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
def test_test_module(self):
self.useFixture(SampleTestFixture())
stream = StringIO()
dist = Distribution()
dist.script_name = 'setup.py'
dist.script_args = ['test']
dist.cmdclass = {'test': TestCommand}
dist.command_options = {
'test': {'test_module': ('command line', 'testtools.runexample')}}
cmd = dist.reinitialize_command('test')
cmd.runner.stdout = stream
dist.run_command('test')
self.assertEqual("""Tests running...
Ran 2 tests in 0.000s
OK
""", stream.getvalue())
def _build_impl(self):
dist = Distribution({
"script_name": None,
"script_args": ["build_ext"]
})
dist.ext_modules = cythonize([self.extension])
dist.include_dirs = []
dist.cmdclass = {'build_ext': custom_build_ext}
build = dist.get_command_obj('build')
# following the convention of cython's pyxbuild and naming
# base directory "_pyxbld"
build.build_base = join(self.CYMJ_DIR_PATH, 'generated',
'_pyxbld_%s' % self.__class__.__name__)
dist.parse_command_line()
obj_build_ext = dist.get_command_obj("build_ext")
dist.run_commands()
built_so_file_path, = obj_build_ext.get_outputs()
return built_so_file_path
def build(self):
dist = Distribution({
"script_name": None,
"script_args": ["build_ext"]
})
dist.ext_modules = cythonize([self.extension])
dist.include_dirs = []
dist.cmdclass = {'build_ext': custom_build_ext}
build = dist.get_command_obj('build')
# following the convention of cython's pyxbuild and naming
# base directory "_pyxbld"
build.build_base = join(self.CYMJ_DIR_PATH, 'generated',
'_pyxbld_%s' % self.__class__.__name__)
dist.parse_command_line()
obj_build_ext = dist.get_command_obj("build_ext")
dist.run_commands()
so_file_path, = obj_build_ext.get_outputs()
return so_file_path
def test_test_module(self):
self.useFixture(SampleTestFixture())
stdout = self.useFixture(fixtures.StringStream('stdout'))
dist = Distribution()
dist.script_name = 'setup.py'
dist.script_args = ['test']
dist.cmdclass = {'test': TestCommand}
dist.command_options = {
'test': {'test_module': ('command line', 'testtools.runexample')}}
cmd = dist.reinitialize_command('test')
with fixtures.MonkeyPatch('sys.stdout', stdout.stream):
dist.run_command('test')
self.assertThat(
stdout.getDetails()['stdout'].as_text(),
MatchesRegex(_u("""Tests running...
Ran 2 tests in \\d.\\d\\d\\ds
OK
""")))
def test_test_module(self):
self.useFixture(SampleTestFixture())
stdout = self.useFixture(fixtures.StringStream('stdout'))
dist = Distribution()
dist.script_name = 'setup.py'
dist.script_args = ['test']
dist.cmdclass = {'test': TestCommand}
dist.command_options = {
'test': {'test_module': ('command line', 'testtools.runexample')}}
with fixtures.MonkeyPatch('sys.stdout', stdout.stream):
cmd = dist.reinitialize_command('test')
dist.run_command('test')
self.assertThat(
stdout.getDetails()['stdout'].as_text(),
MatchesRegex("""Tests running...
Ran 2 tests in \\d.\\d\\d\\ds
OK
"""))
def test_test_module(self):
self.useFixture(SampleTestFixture())
stream = BytesIO()
dist = Distribution()
dist.script_name = 'setup.py'
dist.script_args = ['test']
dist.cmdclass = {'test': TestCommand}
dist.command_options = {
'test': {'test_module': ('command line', 'testtools.runexample')}}
cmd = dist.reinitialize_command('test')
cmd.runner.stdout = stream
dist.run_command('test')
self.assertThat(
stream.getvalue(),
MatchesRegex(_b("""Tests running...
Ran 2 tests in \\d.\\d\\d\\ds
OK
""")))
def c_to_dll(filename, ext = None, force_rebuild = 0,
build_in_temp=False, cbuild_dir=None, setup_args={},
reload_support=False, inplace=False):
"""Compile a C file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
path, name = os.path.split(os.path.abspath(filename))
if not ext:
modname, extension = os.path.splitext(name)
assert extension in (".c", ".py"), extension
if not HAS_CYTHON:
filename = filename[:-len(extension)] + '.c'
ext = Extension(name=modname, sources=[filename])
if not cbuild_dir:
cbuild_dir = os.path.join(path, "_cbld")
package_base_dir = path
for package_name in ext.name.split('.')[-2::-1]:
package_base_dir, pname = os.path.split(package_base_dir)
if pname != package_name:
# something is wrong - package path doesn't match file path
package_base_dir = None
break
script_args=setup_args.get("script_args",[])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
if inplace and package_base_dir:
args.extend(['--build-lib', package_base_dir])
if ext.name == '__init__' or ext.name.endswith('.__init__'):
# package => provide __path__ early
if not hasattr(ext, 'cython_directives'):
ext.cython_directives = {'set_initial_path' : 'SOURCEFILE'}
elif 'set_initial_path' not in ext.cython_directives:
ext.cython_directives['set_initial_path'] = 'SOURCEFILE'
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
sargs = setup_args.copy()
sargs.update({
"script_name": None,
"script_args": args + script_args,
})
# late import, in case setuptools replaced it
from distutils.dist import Distribution
dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
if HAS_CYTHON:
dist.cmdclass = {'build_ext': build_ext}
build = dist.get_command_obj('build')
build.build_base = cbuild_dir
cfgfiles = dist.find_config_files()
dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
raise
if DEBUG:
print("options (after parsing command line):")
dist.dump_option_dicts()
assert ok
try:
obj_build_ext = dist.get_command_obj("build_ext")
dist.run_commands()
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) )
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
basename + '.reload%s'%count)
try:
import shutil # late import / reload_support is: debugging
try:
# Try to unlink first --- if the .so file
# is mmapped by another process,
# overwriting its contents corrupts the
# loaded image (on Linux) and crashes the
# other process. On Windows, unlinking an
# open file just fails.
if os.path.isfile(r_path):
os.unlink(r_path)
except OSError:
continue
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum"%org_path)
_reloads[org_path]=(timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
exc = sys.exc_info()[1]
error = grok_environment_error(exc)
if DEBUG:
sys.stderr.write(error + "\n")
raise
def pyx_to_dll(filename,
ext=None,
force_rebuild=0,
build_in_temp=False,
pyxbuild_dir=None,
setup_args={},
reload_support=False,
inplace=False):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(
filename), "Could not find %s" % os.path.abspath(filename)
path, name = os.path.split(os.path.abspath(filename))
if not ext:
modname, extension = os.path.splitext(name)
assert extension in (".pyx", ".py"), extension
if not HAS_CYTHON:
filename = filename[:-len(extension)] + '.c'
ext = Extension(name=modname, sources=[filename])
if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld")
package_base_dir = path
for package_name in ext.name.split('.')[-2::-1]:
package_base_dir, pname = os.path.split(package_base_dir)
if pname != package_name:
# something is wrong - package path doesn't match file path
package_base_dir = None
break
script_args = setup_args.get("script_args", [])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
if inplace and package_base_dir:
args.extend(['--build-lib', package_base_dir])
if ext.name == '__init__' or ext.name.endswith('.__init__'):
# package => provide __path__ early
if not hasattr(ext, 'cython_directives'):
ext.cython_directives = {'set_initial_path': 'SOURCEFILE'}
elif 'set_initial_path' not in ext.cython_directives:
ext.cython_directives['set_initial_path'] = 'SOURCEFILE'
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
sargs = setup_args.copy()
sargs.update({"script_name": None, "script_args": args + script_args})
dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
if HAS_CYTHON:
dist.cmdclass = {'build_ext': build_ext}
build = dist.get_command_obj('build')
build.build_base = pyxbuild_dir
cfgfiles = dist.find_config_files()
dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
raise
if DEBUG:
print("options (after parsing command line):")
dist.dump_option_dicts()
assert ok
try:
obj_build_ext = dist.get_command_obj("build_ext")
dist.run_commands()
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(
org_path, (None, None, 0))
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
basename + '.reload%s' % count)
try:
import shutil # late import / reload_support is: debugging
try:
# Try to unlink first --- if the .so file
# is mmapped by another process,
# overwriting its contents corrupts the
# loaded image (on Linux) and crashes the
# other process. On Windows, unlinking an
# open file just fails.
if os.path.isfile(r_path):
os.unlink(r_path)
except OSError:
continue
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum" %
org_path)
_reloads[org_path] = (timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
exc = sys.exc_info()[1]
error = grok_environment_error(exc)
if DEBUG:
sys.stderr.write(error + "\n")
raise
def pyx_to_dll(
filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False
):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
path, name = os.path.split(filename)
if not ext:
modname, extension = os.path.splitext(name)
assert extension in (".pyx", ".py"), extension
if not HAS_CYTHON:
filename = filename[: -len(extension)] + ".c"
ext = Extension(name=modname, sources=[filename])
if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld")
script_args = setup_args.get("script_args", [])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
sargs = setup_args.copy()
sargs.update({"script_name": None, "script_args": args + script_args})
dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
if HAS_CYTHON:
dist.cmdclass = {"build_ext": build_ext}
build = dist.get_command_obj("build")
build.build_base = pyxbuild_dir
config_files = dist.find_config_files()
try:
config_files.remove("setup.cfg")
except ValueError:
pass
dist.parse_config_files(config_files)
cfgfiles = dist.find_config_files()
try:
cfgfiles.remove("setup.cfg")
except ValueError:
pass
dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
raise
if DEBUG:
print("options (after parsing command line):")
dist.dump_option_dicts()
assert ok
try:
dist.run_commands()
obj_build_ext = dist.get_command_obj("build_ext")
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(org_path, (None, None, 0))
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib, basename + ".reload%s" % count)
try:
import shutil # late import / reload_support is: debugging
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum" % org_path)
_reloads[org_path] = (timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
exc = sys.exc_info()[1]
error = grok_environment_error(exc)
if DEBUG:
sys.stderr.write(error + "\n")
raise
def pyx_to_dll(filename,
ext=None,
force_rebuild=0,
build_in_temp=False,
pyxbuild_dir=None,
setup_args={},
reload_support=False):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(
filename), "Could not find %s" % os.path.abspath(filename)
path, name = os.path.split(filename)
if not ext:
modname, extension = os.path.splitext(name)
assert extension in (".pyx", ".py"), extension
if not HAS_CYTHON:
filename = filename[:-len(extension)] + '.c'
ext = Extension(name=modname, sources=[filename])
if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld")
script_args = setup_args.get("script_args", [])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
args = [quiet, "build_ext"]
if force_rebuild:
args.append("--force")
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
sargs = setup_args.copy()
sargs.update({"script_name": None, "script_args": args + script_args})
dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
if HAS_CYTHON:
dist.cmdclass = {'build_ext': build_ext}
build = dist.get_command_obj('build')
build.build_base = pyxbuild_dir
config_files = dist.find_config_files()
try:
config_files.remove('setup.cfg')
except ValueError:
pass
dist.parse_config_files(config_files)
cfgfiles = dist.find_config_files()
try:
cfgfiles.remove('setup.cfg')
except ValueError:
pass
dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
raise
if DEBUG:
print("options (after parsing command line):")
dist.dump_option_dicts()
assert ok
try:
dist.run_commands()
obj_build_ext = dist.get_command_obj("build_ext")
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(
org_path, (None, None, 0))
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
basename + '.reload%s' % count)
try:
import shutil # late import / reload_support is: debugging
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum" %
org_path)
_reloads[org_path] = (timestamp, so_path, count)
return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
exc = sys.exc_info()[1]
error = grok_environment_error(exc)
if DEBUG:
sys.stderr.write(error + "\n")
raise