def pyx_to_dll(filename, ext=None, force_rebuild=0): """Compile a PYX file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists(filename) path, name = os.path.split(filename) if not ext: modname, extension = os.path.splitext(name) assert extension == ".pyx", extension ext = Extension(name=modname, sources=[filename]) if DEBUG: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") dist = Distribution({"script_name": None, "script_args": args}) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) dist.cmdclass = {'build_ext': build_ext} build = dist.get_command_obj('build') build.build_base = os.path.join(path, "_pyxbld") try: ok = dist.parse_command_line() except DistutilsArgError, msg: raise
def pyx_to_dll(filename, ext = None, force_rebuild = 0): """Compile a PYX file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists(filename) path, name = os.path.split(filename) if not ext: modname, extension = os.path.splitext(name) assert extension == ".pyx", extension ext = Extension(name=modname, sources=[filename]) if DEBUG: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") dist = Distribution({"script_name": None, "script_args": args}) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) dist.cmdclass = {'build_ext': build_ext} build = dist.get_command_obj('build') build.build_base = os.path.join(path, "_pyxbld") try: ok = dist.parse_command_line() except DistutilsArgError, msg: raise
def build_cython_extension(py_or_pyx_file_path, cython_force_rebuild=True): """ Build a cython extension from a `.py` or `.pyx` file - build will be done in a sub-folder named `_pyxbld` in the py_or_pyx_file_path :param py_or_pyx_file_path: (str) path to a `.py` or `.pyx` file :param cython_force_rebuild: (bool) If True the cython extension is rebuild even if it was already build :return: (tuple) cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path """ module_dir = path_dirname(py_or_pyx_file_path) module__cython_name = path_splitext(path_basename(py_or_pyx_file_path))[0] cython_module_c_file_path = path_join(module_dir, module__cython_name + '.c') cython_build_dir_path = path_join(module_dir, '_pyxbld') args = ['--quiet', 'build_ext', '--build-lib', module_dir] if cython_force_rebuild: args.append('--force') dist = Distribution({'script_name': None, 'script_args': args}) dist.ext_modules = [ Extension(name=module__cython_name, sources=[py_or_pyx_file_path]) ] dist.cmdclass = {'build_ext': cython_build_ext} build = dist.get_command_obj('build') build.build_base = cython_build_dir_path try: dist.parse_command_line() except DistutilsArgError as err: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' DistutilsArgError: <{}>'.format(err) ]) try: obj_build_ext = dist.get_command_obj('build_ext') dist.run_commands() cython_extension_module_path = obj_build_ext.get_outputs()[0] if path_dirname(py_or_pyx_file_path) != module_dir: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' <module_dir> differs from final <cython_module_dir>', ' module_dir: <{}>'.format(module_dir), ' cython_module_dir: <{}>'.format( path_dirname(py_or_pyx_file_path)) ]) except Exception as err: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' Exception: <{}>'.format(err) ]) return cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
def build_cython_extension(py_or_pyx_file_path, cython_force_rebuild=True): """ Build a cython extension from a `.py` or `.pyx` file - build will be done in a sub-folder named `_pyxbld` in the py_or_pyx_file_path :param py_or_pyx_file_path: (str) path to a `.py` or `.pyx` file :param cython_force_rebuild: (bool) If True the cython extension is rebuild even if it was already build :return: (tuple) cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path """ module_dir = path_dirname(py_or_pyx_file_path) module__cython_name = path_splitext(path_basename(py_or_pyx_file_path))[0] cython_module_c_file_path = path_join(module_dir, module__cython_name + '.c') cython_build_dir_path = path_join(module_dir, '_pyxbld') args = ['--quiet', 'build_ext', '--build-lib', module_dir] if cython_force_rebuild: args.append('--force') dist = Distribution({'script_name': None, 'script_args': args}) dist.ext_modules = [Extension(name=module__cython_name, sources=[py_or_pyx_file_path])] dist.cmdclass = {'build_ext': cython_build_ext} build = dist.get_command_obj('build') build.build_base = cython_build_dir_path try: dist.parse_command_line() except DistutilsArgError as err: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' DistutilsArgError: <{}>'.format(err) ]) try: obj_build_ext = dist.get_command_obj('build_ext') dist.run_commands() cython_extension_module_path = obj_build_ext.get_outputs()[0] if path_dirname(py_or_pyx_file_path) != module_dir: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' <module_dir> differs from final <cython_module_dir>', ' module_dir: <{}>'.format(module_dir), ' cython_module_dir: <{}>'.format(path_dirname(py_or_pyx_file_path)) ]) except Exception as err: raise Err('utils.build_cython_extension', [ 'py_or_pyx_file_path: <{}>'.format(py_or_pyx_file_path), ' Exception: <{}>'.format(err) ]) return cython_extension_module_path, cython_module_c_file_path, cython_build_dir_path
def test_test_module(self): self.useFixture(SampleTestFixture()) stream = StringIO() dist = Distribution() dist.script_name = 'setup.py' dist.script_args = ['test'] dist.cmdclass = {'test': TestCommand} dist.command_options = { 'test': {'test_module': ('command line', 'testtools.runexample')}} cmd = dist.reinitialize_command('test') cmd.runner.stdout = stream dist.run_command('test') self.assertEqual("""Tests running... Ran 2 tests in 0.000s OK """, stream.getvalue())
def _build_impl(self): dist = Distribution({ "script_name": None, "script_args": ["build_ext"] }) dist.ext_modules = cythonize([self.extension]) dist.include_dirs = [] dist.cmdclass = {'build_ext': custom_build_ext} build = dist.get_command_obj('build') # following the convention of cython's pyxbuild and naming # base directory "_pyxbld" build.build_base = join(self.CYMJ_DIR_PATH, 'generated', '_pyxbld_%s' % self.__class__.__name__) dist.parse_command_line() obj_build_ext = dist.get_command_obj("build_ext") dist.run_commands() built_so_file_path, = obj_build_ext.get_outputs() return built_so_file_path
def build(self): dist = Distribution({ "script_name": None, "script_args": ["build_ext"] }) dist.ext_modules = cythonize([self.extension]) dist.include_dirs = [] dist.cmdclass = {'build_ext': custom_build_ext} build = dist.get_command_obj('build') # following the convention of cython's pyxbuild and naming # base directory "_pyxbld" build.build_base = join(self.CYMJ_DIR_PATH, 'generated', '_pyxbld_%s' % self.__class__.__name__) dist.parse_command_line() obj_build_ext = dist.get_command_obj("build_ext") dist.run_commands() so_file_path, = obj_build_ext.get_outputs() return so_file_path
def test_test_module(self): self.useFixture(SampleTestFixture()) stdout = self.useFixture(fixtures.StringStream('stdout')) dist = Distribution() dist.script_name = 'setup.py' dist.script_args = ['test'] dist.cmdclass = {'test': TestCommand} dist.command_options = { 'test': {'test_module': ('command line', 'testtools.runexample')}} cmd = dist.reinitialize_command('test') with fixtures.MonkeyPatch('sys.stdout', stdout.stream): dist.run_command('test') self.assertThat( stdout.getDetails()['stdout'].as_text(), MatchesRegex(_u("""Tests running... Ran 2 tests in \\d.\\d\\d\\ds OK """)))
def test_test_module(self): self.useFixture(SampleTestFixture()) stdout = self.useFixture(fixtures.StringStream('stdout')) dist = Distribution() dist.script_name = 'setup.py' dist.script_args = ['test'] dist.cmdclass = {'test': TestCommand} dist.command_options = { 'test': {'test_module': ('command line', 'testtools.runexample')}} with fixtures.MonkeyPatch('sys.stdout', stdout.stream): cmd = dist.reinitialize_command('test') dist.run_command('test') self.assertThat( stdout.getDetails()['stdout'].as_text(), MatchesRegex("""Tests running... Ran 2 tests in \\d.\\d\\d\\ds OK """))
def test_test_module(self): self.useFixture(SampleTestFixture()) stream = BytesIO() dist = Distribution() dist.script_name = 'setup.py' dist.script_args = ['test'] dist.cmdclass = {'test': TestCommand} dist.command_options = { 'test': {'test_module': ('command line', 'testtools.runexample')}} cmd = dist.reinitialize_command('test') cmd.runner.stdout = stream dist.run_command('test') self.assertThat( stream.getvalue(), MatchesRegex(_b("""Tests running... Ran 2 tests in \\d.\\d\\d\\ds OK """)))
def c_to_dll(filename, ext = None, force_rebuild = 0, build_in_temp=False, cbuild_dir=None, setup_args={}, reload_support=False, inplace=False): """Compile a C file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename) path, name = os.path.split(os.path.abspath(filename)) if not ext: modname, extension = os.path.splitext(name) assert extension in (".c", ".py"), extension if not HAS_CYTHON: filename = filename[:-len(extension)] + '.c' ext = Extension(name=modname, sources=[filename]) if not cbuild_dir: cbuild_dir = os.path.join(path, "_cbld") package_base_dir = path for package_name in ext.name.split('.')[-2::-1]: package_base_dir, pname = os.path.split(package_base_dir) if pname != package_name: # something is wrong - package path doesn't match file path package_base_dir = None break script_args=setup_args.get("script_args",[]) if DEBUG or "--verbose" in script_args: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") if inplace and package_base_dir: args.extend(['--build-lib', package_base_dir]) if ext.name == '__init__' or ext.name.endswith('.__init__'): # package => provide __path__ early if not hasattr(ext, 'cython_directives'): ext.cython_directives = {'set_initial_path' : 'SOURCEFILE'} elif 'set_initial_path' not in ext.cython_directives: ext.cython_directives['set_initial_path'] = 'SOURCEFILE' if HAS_CYTHON and build_in_temp: args.append("--pyrex-c-in-temp") sargs = setup_args.copy() sargs.update({ "script_name": None, "script_args": args + script_args, }) # late import, in case setuptools replaced it from distutils.dist import Distribution dist = Distribution(sargs) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) if HAS_CYTHON: dist.cmdclass = {'build_ext': build_ext} build = dist.get_command_obj('build') build.build_base = cbuild_dir cfgfiles = dist.find_config_files() dist.parse_config_files(cfgfiles) try: ok = dist.parse_command_line() except DistutilsArgError: raise if DEBUG: print("options (after parsing command line):") dist.dump_option_dicts() assert ok try: obj_build_ext = dist.get_command_obj("build_ext") dist.run_commands() so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path # when --inplace ; see http://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) if reload_support: org_path = so_path timestamp = os.path.getmtime(org_path) global _reloads last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) ) if last_timestamp == timestamp: so_path = last_path else: basename = os.path.basename(org_path) while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, basename + '.reload%s'%count) try: import shutil # late import / reload_support is: debugging try: # Try to unlink first --- if the .so file # is mmapped by another process, # overwriting its contents corrupts the # loaded image (on Linux) and crashes the # other process. On Windows, unlinking an # open file just fails. if os.path.isfile(r_path): os.unlink(r_path) except OSError: continue shutil.copy2(org_path, r_path) so_path = r_path except IOError: continue break else: # used up all 100 slots raise ImportError("reload count for %s reached maximum"%org_path) _reloads[org_path]=(timestamp, so_path, count) return so_path except KeyboardInterrupt: sys.exit(1) except (IOError, os.error): exc = sys.exc_info()[1] error = grok_environment_error(exc) if DEBUG: sys.stderr.write(error + "\n") raise
def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False, inplace=False): """Compile a PYX file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists( filename), "Could not find %s" % os.path.abspath(filename) path, name = os.path.split(os.path.abspath(filename)) if not ext: modname, extension = os.path.splitext(name) assert extension in (".pyx", ".py"), extension if not HAS_CYTHON: filename = filename[:-len(extension)] + '.c' ext = Extension(name=modname, sources=[filename]) if not pyxbuild_dir: pyxbuild_dir = os.path.join(path, "_pyxbld") package_base_dir = path for package_name in ext.name.split('.')[-2::-1]: package_base_dir, pname = os.path.split(package_base_dir) if pname != package_name: # something is wrong - package path doesn't match file path package_base_dir = None break script_args = setup_args.get("script_args", []) if DEBUG or "--verbose" in script_args: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") if inplace and package_base_dir: args.extend(['--build-lib', package_base_dir]) if ext.name == '__init__' or ext.name.endswith('.__init__'): # package => provide __path__ early if not hasattr(ext, 'cython_directives'): ext.cython_directives = {'set_initial_path': 'SOURCEFILE'} elif 'set_initial_path' not in ext.cython_directives: ext.cython_directives['set_initial_path'] = 'SOURCEFILE' if HAS_CYTHON and build_in_temp: args.append("--pyrex-c-in-temp") sargs = setup_args.copy() sargs.update({"script_name": None, "script_args": args + script_args}) dist = Distribution(sargs) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) if HAS_CYTHON: dist.cmdclass = {'build_ext': build_ext} build = dist.get_command_obj('build') build.build_base = pyxbuild_dir cfgfiles = dist.find_config_files() dist.parse_config_files(cfgfiles) try: ok = dist.parse_command_line() except DistutilsArgError: raise if DEBUG: print("options (after parsing command line):") dist.dump_option_dicts() assert ok try: obj_build_ext = dist.get_command_obj("build_ext") dist.run_commands() so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path # when --inplace ; see http://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) if reload_support: org_path = so_path timestamp = os.path.getmtime(org_path) global _reloads last_timestamp, last_path, count = _reloads.get( org_path, (None, None, 0)) if last_timestamp == timestamp: so_path = last_path else: basename = os.path.basename(org_path) while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, basename + '.reload%s' % count) try: import shutil # late import / reload_support is: debugging try: # Try to unlink first --- if the .so file # is mmapped by another process, # overwriting its contents corrupts the # loaded image (on Linux) and crashes the # other process. On Windows, unlinking an # open file just fails. if os.path.isfile(r_path): os.unlink(r_path) except OSError: continue shutil.copy2(org_path, r_path) so_path = r_path except IOError: continue break else: # used up all 100 slots raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path] = (timestamp, so_path, count) return so_path except KeyboardInterrupt: sys.exit(1) except (IOError, os.error): exc = sys.exc_info()[1] error = grok_environment_error(exc) if DEBUG: sys.stderr.write(error + "\n") raise
def pyx_to_dll( filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False ): """Compile a PYX file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename) path, name = os.path.split(filename) if not ext: modname, extension = os.path.splitext(name) assert extension in (".pyx", ".py"), extension if not HAS_CYTHON: filename = filename[: -len(extension)] + ".c" ext = Extension(name=modname, sources=[filename]) if not pyxbuild_dir: pyxbuild_dir = os.path.join(path, "_pyxbld") script_args = setup_args.get("script_args", []) if DEBUG or "--verbose" in script_args: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") if HAS_CYTHON and build_in_temp: args.append("--pyrex-c-in-temp") sargs = setup_args.copy() sargs.update({"script_name": None, "script_args": args + script_args}) dist = Distribution(sargs) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) if HAS_CYTHON: dist.cmdclass = {"build_ext": build_ext} build = dist.get_command_obj("build") build.build_base = pyxbuild_dir config_files = dist.find_config_files() try: config_files.remove("setup.cfg") except ValueError: pass dist.parse_config_files(config_files) cfgfiles = dist.find_config_files() try: cfgfiles.remove("setup.cfg") except ValueError: pass dist.parse_config_files(cfgfiles) try: ok = dist.parse_command_line() except DistutilsArgError: raise if DEBUG: print("options (after parsing command line):") dist.dump_option_dicts() assert ok try: dist.run_commands() obj_build_ext = dist.get_command_obj("build_ext") so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path # when --inplace ; see http://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) if reload_support: org_path = so_path timestamp = os.path.getmtime(org_path) global _reloads last_timestamp, last_path, count = _reloads.get(org_path, (None, None, 0)) if last_timestamp == timestamp: so_path = last_path else: basename = os.path.basename(org_path) while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, basename + ".reload%s" % count) try: import shutil # late import / reload_support is: debugging shutil.copy2(org_path, r_path) so_path = r_path except IOError: continue break else: # used up all 100 slots raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path] = (timestamp, so_path, count) return so_path except KeyboardInterrupt: sys.exit(1) except (IOError, os.error): exc = sys.exc_info()[1] error = grok_environment_error(exc) if DEBUG: sys.stderr.write(error + "\n") raise
def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False): """Compile a PYX file to a DLL and return the name of the generated .so or .dll .""" assert os.path.exists( filename), "Could not find %s" % os.path.abspath(filename) path, name = os.path.split(filename) if not ext: modname, extension = os.path.splitext(name) assert extension in (".pyx", ".py"), extension if not HAS_CYTHON: filename = filename[:-len(extension)] + '.c' ext = Extension(name=modname, sources=[filename]) if not pyxbuild_dir: pyxbuild_dir = os.path.join(path, "_pyxbld") script_args = setup_args.get("script_args", []) if DEBUG or "--verbose" in script_args: quiet = "--verbose" else: quiet = "--quiet" args = [quiet, "build_ext"] if force_rebuild: args.append("--force") if HAS_CYTHON and build_in_temp: args.append("--pyrex-c-in-temp") sargs = setup_args.copy() sargs.update({"script_name": None, "script_args": args + script_args}) dist = Distribution(sargs) if not dist.ext_modules: dist.ext_modules = [] dist.ext_modules.append(ext) if HAS_CYTHON: dist.cmdclass = {'build_ext': build_ext} build = dist.get_command_obj('build') build.build_base = pyxbuild_dir config_files = dist.find_config_files() try: config_files.remove('setup.cfg') except ValueError: pass dist.parse_config_files(config_files) cfgfiles = dist.find_config_files() try: cfgfiles.remove('setup.cfg') except ValueError: pass dist.parse_config_files(cfgfiles) try: ok = dist.parse_command_line() except DistutilsArgError: raise if DEBUG: print("options (after parsing command line):") dist.dump_option_dicts() assert ok try: dist.run_commands() obj_build_ext = dist.get_command_obj("build_ext") so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path # when --inplace ; see http://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) if reload_support: org_path = so_path timestamp = os.path.getmtime(org_path) global _reloads last_timestamp, last_path, count = _reloads.get( org_path, (None, None, 0)) if last_timestamp == timestamp: so_path = last_path else: basename = os.path.basename(org_path) while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, basename + '.reload%s' % count) try: import shutil # late import / reload_support is: debugging shutil.copy2(org_path, r_path) so_path = r_path except IOError: continue break else: # used up all 100 slots raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path] = (timestamp, so_path, count) return so_path except KeyboardInterrupt: sys.exit(1) except (IOError, os.error): exc = sys.exc_info()[1] error = grok_environment_error(exc) if DEBUG: sys.stderr.write(error + "\n") raise