def _check_append_library(libraries, item): for libitem in libraries: if is_sequence(libitem): if is_sequence(item): if item[0] == libitem[0]: if item[1] is libitem[1]: return warnings.warn("[0] libraries list contains %r with" " different build_info" % (item[0], ), stacklevel=2) break else: if item == libitem[0]: warnings.warn("[1] libraries list contains %r with" " no build_info" % (item[0], ), stacklevel=2) break else: if is_sequence(item): if item[0] == libitem: warnings.warn("[2] libraries list contains %r with" " no build_info" % (item[0], ), stacklevel=2) break else: if item == libitem: return libraries.append(item)
def _exec_command(command, use_shell=None, use_tee=None, **env): """ Internal workhorse for exec_command(). """ if use_shell is None: use_shell = os.name == 'posix' if use_tee is None: use_tee = os.name == 'posix' if os.name == 'posix' and use_shell: # On POSIX, subprocess always uses /bin/sh, override sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): command = [sh, '-c', ' '.join(command)] else: command = [sh, '-c', command] use_shell = False elif os.name == 'nt' and is_sequence(command): # On Windows, join the string for CreateProcess() ourselves as # subprocess does it a bit differently command = ' '.join(_quote_arg(arg) for arg in command) # Inherit environment by default env = env or None try: # universal_newlines is set to False so that communicate() # will return bytes. We need to decode the output ourselves # so that Python will not raise a UnicodeDecodeError when # it encounters an invalid character; rather, we simply replace it proc = subprocess.Popen(command, shell=use_shell, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=False) except EnvironmentError: # Return 127, as os.spawn*() and /bin/sh do return 127, '' text, err = proc.communicate() text = text.decode(locale.getpreferredencoding(False), errors='replace') text = text.replace('\r\n', '\n') # Another historical oddity if text[-1:] == '\n': text = text[:-1] # stdio uses bytes in python 2, so to avoid issues, we simply # remove all non-ascii characters if sys.version_info < (3, 0): text = text.encode('ascii', errors='replace') if use_tee and text: print(text) return proc.returncode, text
def build_py_modules_sources(self): if not self.py_modules: return log.info('building py_modules sources') new_py_modules = [] for source in self.py_modules: if is_sequence(source) and len(source) == 3: package, module_base, source = source if self.inplace: build_dir = self.get_package_dir(package) else: build_dir = os.path.join(self.build_src, os.path.join(*package.split('.'))) if hasattr(source, '__call__'): target = os.path.join(build_dir, module_base + '.py') source = source(target) if source is None: continue modules = [(package, module_base, source)] if package not in self.py_modules_dict: self.py_modules_dict[package] = [] self.py_modules_dict[package] += modules else: new_py_modules.append(source) self.py_modules[:] = new_py_modules
def CCompiler_spawn(self, cmd, display=None): """ Execute a command in a sub-process. Parameters ---------- cmd : str The command to execute. display : str or sequence of str, optional The text to add to the log file kept by `numpy.distutils`. If not given, `display` is equal to `cmd`. Returns ------- None Raises ------ DistutilsExecError If the command failed, i.e. the exit status was not 0. """ if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) s, o = exec_command(cmd) if s: if is_sequence(cmd): cmd = ' '.join(list(cmd)) try: print(o) except UnicodeError: # When installing through pip, `o` can contain non-ascii chars pass if re.search('Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' else: msg = '' raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): library_dirs = quote_args(library_dirs) runtime_library_dirs = quote_args(runtime_library_dirs) r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] for i in r: if is_sequence(i): lib_opts.extend(list(i)) else: lib_opts.append(i) return lib_opts
def generate_sources(self, sources, extension): new_sources = [] func_sources = [] for source in sources: if is_string(source): new_sources.append(source) else: func_sources.append(source) if not func_sources: return new_sources if self.inplace and not is_sequence(extension): build_dir = self.ext_target_dir else: if is_sequence(extension): name = extension[0] # if 'include_dirs' not in extension[1]: # extension[1]['include_dirs'] = [] # incl_dirs = extension[1]['include_dirs'] else: name = extension.name # incl_dirs = extension.include_dirs #if self.build_src not in incl_dirs: # incl_dirs.append(self.build_src) build_dir = os.path.join(*([self.build_src]\ +name.split('.')[:-1])) self.mkpath(build_dir) for func in func_sources: source = func(extension, build_dir) if not source: continue if is_sequence(source): [log.info(" adding '%s' to sources." % (s, )) for s in source] new_sources.extend(source) else: log.info(" adding '%s' to sources." % (source, )) new_sources.append(source) return new_sources
def _check_append_ext_library(libraries, lib_name, build_info): for item in libraries: if is_sequence(item): if item[0] == lib_name: if item[1] is build_info: return warnings.warn("[3] libraries list contains %r with" " different build_info" % (lib_name, ), stacklevel=2) break elif item == lib_name: warnings.warn("[4] libraries list contains %r with" " no build_info" % (lib_name, ), stacklevel=2) break libraries.append((lib_name, build_info))
def template_sources(self, sources, extension): new_sources = [] if is_sequence(extension): depends = extension[1].get('depends') include_dirs = extension[1].get('include_dirs') else: depends = extension.depends include_dirs = extension.include_dirs for source in sources: (base, ext) = os.path.splitext(source) if ext == '.src': # Template file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) self.mkpath(target_dir) target_file = os.path.join(target_dir, os.path.basename(base)) if (self.force or newer_group([source] + depends, target_file)): if _f_pyf_ext_match(base): log.info("from_template:> %s" % (target_file)) outstr = process_f_file(source) else: log.info("conv_template:> %s" % (target_file)) outstr = process_c_file(source) fid = open(target_file, 'w') fid.write(outstr) fid.close() if _header_ext_match(target_file): d = os.path.dirname(target_file) if d not in include_dirs: log.info(" adding '%s' to include_dirs." % (d)) include_dirs.append(d) new_sources.append(target_file) else: new_sources.append(source) return new_sources
def f2py_sources(self, sources, extension): new_sources = [] f2py_sources = [] f_sources = [] f2py_targets = {} target_dirs = [] ext_name = extension.name.split('.')[-1] skip_f2py = 0 for source in sources: (base, ext) = os.path.splitext(source) if ext == '.pyf': # F2PY interface file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) if os.path.isfile(source): name = get_f2py_modulename(source) if name != ext_name: raise DistutilsSetupError( 'mismatch of extension names: %s ' 'provides %r but expected %r' % (source, name, ext_name)) target_file = os.path.join(target_dir, name + 'module.c') else: log.debug(' source %s does not exist: skipping f2py\'ing.' \ % (source)) name = ext_name skip_f2py = 1 target_file = os.path.join(target_dir, name + 'module.c') if not os.path.isfile(target_file): log.warn(' target %s does not exist:\n '\ 'Assuming %smodule.c was generated with '\ '"build_src --inplace" command.' \ % (target_file, name)) target_dir = os.path.dirname(base) target_file = os.path.join(target_dir, name + 'module.c') if not os.path.isfile(target_file): raise DistutilsSetupError("%r missing" % (target_file, )) log.info(' Yes! Using %r as up-to-date target.' \ % (target_file)) target_dirs.append(target_dir) f2py_sources.append(source) f2py_targets[source] = target_file new_sources.append(target_file) elif fortran_ext_match(ext): f_sources.append(source) else: new_sources.append(source) if not (f2py_sources or f_sources): return new_sources for d in target_dirs: self.mkpath(d) f2py_options = extension.f2py_options + self.f2py_opts if self.distribution.libraries: for name, build_info in self.distribution.libraries: if name in extension.libraries: f2py_options.extend(build_info.get('f2py_options', [])) log.info("f2py options: %s" % (f2py_options)) if f2py_sources: if len(f2py_sources) != 1: raise DistutilsSetupError( 'only one .pyf file is allowed per extension module but got'\ ' more: %r' % (f2py_sources,)) source = f2py_sources[0] target_file = f2py_targets[source] target_dir = os.path.dirname(target_file) or '.' depends = [source] + extension.depends if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) import numpy1.f2py numpy1.f2py.run_main(f2py_options + ['--build-dir', target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) else: #XXX TODO: --inplace support for sdist command if is_sequence(extension): name = extension[0] else: name = extension.name target_dir = os.path.join(*([self.build_src]\ +name.split('.')[:-1])) target_file = os.path.join(target_dir, ext_name + 'module.c') new_sources.append(target_file) depends = f_sources + extension.depends if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) import numpy1.f2py numpy1.f2py.run_main(f2py_options + ['--lower', '--build-dir', target_dir] + \ ['-m', ext_name] + f_sources) else: log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\ % (target_file)) if not os.path.isfile(target_file): raise DistutilsError("f2py target file %r not generated" % (target_file, )) build_dir = os.path.join(self.build_src, target_dir) target_c = os.path.join(build_dir, 'fortranobject.c') target_h = os.path.join(build_dir, 'fortranobject.h') log.info(" adding '%s' to sources." % (target_c)) new_sources.append(target_c) if build_dir not in extension.include_dirs: log.info(" adding '%s' to include_dirs." % (build_dir)) extension.include_dirs.append(build_dir) if not skip_f2py: import numpy1.f2py d = os.path.dirname(numpy1.f2py.__file__) source_c = os.path.join(d, 'src', 'fortranobject.c') source_h = os.path.join(d, 'src', 'fortranobject.h') if newer(source_c, target_c) or newer(source_h, target_h): self.mkpath(os.path.dirname(target_c)) self.copy_file(source_c, target_c) self.copy_file(source_h, target_h) else: if not os.path.isfile(target_c): raise DistutilsSetupError("f2py target_c file %r not found" % (target_c, )) if not os.path.isfile(target_h): raise DistutilsSetupError("f2py target_h file %r not found" % (target_h, )) for name_ext in ['-f2pywrappers.f', '-f2pywrappers2.f90']: filename = os.path.join(target_dir, ext_name + name_ext) if os.path.isfile(filename): log.info(" adding '%s' to sources." % (filename)) f_sources.append(filename) return new_sources + f_sources
def setup(**attr): cmdclass = numpy_cmdclass.copy() new_attr = attr.copy() if 'cmdclass' in new_attr: cmdclass.update(new_attr['cmdclass']) new_attr['cmdclass'] = cmdclass if 'configuration' in new_attr: # To avoid calling configuration if there are any errors # or help request in command in the line. configuration = new_attr.pop('configuration') old_dist = distutils.core._setup_distribution old_stop = distutils.core._setup_stop_after distutils.core._setup_distribution = None distutils.core._setup_stop_after = "commandline" try: dist = setup(**new_attr) finally: distutils.core._setup_distribution = old_dist distutils.core._setup_stop_after = old_stop if dist.help or not _command_line_ok(): # probably displayed help, skip running any commands return dist # create setup dictionary and append to new_attr config = configuration() if hasattr(config, 'todict'): config = config.todict() _dict_append(new_attr, **config) # Move extension source libraries to libraries libraries = [] for ext in new_attr.get('ext_modules', []): new_libraries = [] for item in ext.libraries: if is_sequence(item): lib_name, build_info = item _check_append_ext_library(libraries, lib_name, build_info) new_libraries.append(lib_name) elif is_string(item): new_libraries.append(item) else: raise TypeError("invalid description of extension module " "library %r" % (item, )) ext.libraries = new_libraries if libraries: if 'libraries' not in new_attr: new_attr['libraries'] = [] for item in libraries: _check_append_library(new_attr['libraries'], item) # sources in ext_modules or libraries may contain header files if ('ext_modules' in new_attr or 'libraries' in new_attr) \ and 'headers' not in new_attr: new_attr['headers'] = [] # Use our custom NumpyDistribution class instead of distutils' one new_attr['distclass'] = NumpyDistribution return old_setup(**new_attr)
def _fix_args(args, flag=1): if is_string(args): return args.replace('%', '%%') if flag and is_sequence(args): return tuple([_fix_args(a, flag=0) for a in args]) return args
def build_extension(self, ext): sources = ext.sources if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % ext.name) sources = list(sources) if not sources: return fullname = self.get_ext_fullname(ext.name) if self.inplace: modpath = fullname.split('.') package = '.'.join(modpath[0:-1]) base = modpath[-1] build_py = self.get_finalized_command('build_py') package_dir = build_py.get_package_dir(package) ext_filename = os.path.join(package_dir, self.get_ext_filename(base)) else: ext_filename = os.path.join(self.build_lib, self.get_ext_filename(fullname)) depends = sources + ext.depends if not (self.force or newer_group(depends, ext_filename, 'newer')): log.debug("skipping '%s' extension (up-to-date)", ext.name) return else: log.info("building '%s' extension", ext.name) extra_args = ext.extra_compile_args or [] macros = ext.define_macros[:] for undef in ext.undef_macros: macros.append((undef,)) c_sources, cxx_sources, f_sources, fmodule_sources = \ filter_sources(ext.sources) if self.compiler.compiler_type == 'msvc': if cxx_sources: # Needed to compile kiva.agg._agg extension. extra_args.append('/Zm1000') # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] # Set Fortran/C++ compilers for compilation and linking. if ext.language == 'f90': fcompiler = self._f90_compiler elif ext.language == 'f77': fcompiler = self._f77_compiler else: # in case ext.language is c++, for instance fcompiler = self._f90_compiler or self._f77_compiler if fcompiler is not None: fcompiler.extra_f77_compile_args = (ext.extra_f77_compile_args or []) if hasattr( ext, 'extra_f77_compile_args') else [] fcompiler.extra_f90_compile_args = (ext.extra_f90_compile_args or []) if hasattr( ext, 'extra_f90_compile_args') else [] cxx_compiler = self._cxx_compiler # check for the availability of required compilers if cxx_sources and cxx_compiler is None: raise DistutilsError("extension %r has C++ sources" "but no C++ compiler found" % (ext.name)) if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("extension %r has Fortran sources " "but no Fortran compiler found" % (ext.name)) if ext.language in ['f77', 'f90'] and fcompiler is None: self.warn("extension %r has Fortran libraries " "but no Fortran linker found, using default linker" % (ext.name)) if ext.language == 'c++' and cxx_compiler is None: self.warn("extension %r has C++ libraries " "but no C++ linker found, using default linker" % (ext.name)) kws = {'depends': ext.depends} output_dir = self.build_temp include_dirs = ext.include_dirs + get_numpy_include_dirs() c_objects = [] if c_sources: log.info("compiling C sources") c_objects = self.compiler.compile(c_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) extra_postargs = [] f_objects = [] if fmodule_sources: log.info("compiling Fortran 90 module sources") module_dirs = ext.module_dirs[:] module_build_dir = os.path.join( self.build_temp, os.path.dirname( self.get_ext_filename(fullname))) self.mkpath(module_build_dir) if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options( module_dirs, module_build_dir) f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) if fcompiler.module_dir_switch is None: for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) if f_objects and not fcompiler.can_ccompiler_link(self.compiler): unlinkable_fobjects = f_objects objects = c_objects else: unlinkable_fobjects = [] objects = c_objects + f_objects if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] libraries = self.get_libraries(ext)[:] library_dirs = ext.library_dirs[:] linker = self.compiler.link_shared_object # Always use system linker when using MSVC compiler. if self.compiler.compiler_type in ('msvc', 'intelw', 'intelemw'): # expand libraries with fcompiler libraries as we are # not using fcompiler linker self._libs_with_msvc_and_fortran( fcompiler, libraries, library_dirs) elif ext.language in ['f77', 'f90'] and fcompiler is not None: linker = fcompiler.link_shared_object if ext.language == 'c++' and cxx_compiler is not None: linker = cxx_compiler.link_shared_object if fcompiler is not None: objects, libraries = self._process_unlinkable_fobjects( objects, libraries, fcompiler, library_dirs, unlinkable_fobjects) linker(objects, ext_filename, libraries=libraries, library_dirs=library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=ext.language)
def build_a_library(self, build_info, lib_name, libraries): # default compilers compiler = self.compiler fcompiler = self._f_compiler sources = build_info.get('sources') if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) sources = list(sources) c_sources, cxx_sources, f_sources, fmodule_sources \ = filter_sources(sources) requiref90 = not not fmodule_sources or \ build_info.get('language', 'c') == 'f90' # save source type information so that build_ext can use it. source_languages = [] if c_sources: source_languages.append('c') if cxx_sources: source_languages.append('c++') if requiref90: source_languages.append('f90') elif f_sources: source_languages.append('f77') build_info['source_languages'] = source_languages lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) depends = sources + build_info.get('depends', []) if not (self.force or newer_group(depends, lib_file, 'newer')): log.debug("skipping '%s' library (up-to-date)", lib_name) return else: log.info("building '%s' library", lib_name) config_fc = build_info.get('config_fc', {}) if fcompiler is not None and config_fc: log.info('using additional config_fc from setup script ' 'for fortran compiler: %s' % (config_fc, )) from numpy1.distutils.fcompiler import new_fcompiler fcompiler = new_fcompiler(compiler=fcompiler.compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force, requiref90=requiref90, c_compiler=self.compiler) if fcompiler is not None: dist = self.distribution base_config_fc = dist.get_option_dict('config_fc').copy() base_config_fc.update(config_fc) fcompiler.customize(base_config_fc) # check availability of Fortran compilers if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("library %s has Fortran sources" " but no Fortran compiler found" % (lib_name)) if fcompiler is not None: fcompiler.extra_f77_compile_args = build_info.get( 'extra_f77_compile_args') or [] fcompiler.extra_f90_compile_args = build_info.get( 'extra_f90_compile_args') or [] macros = build_info.get('macros') include_dirs = build_info.get('include_dirs') if include_dirs is None: include_dirs = [] extra_postargs = build_info.get('extra_compiler_args') or [] include_dirs.extend(get_numpy_include_dirs()) # where compiled F90 module files are: module_dirs = build_info.get('module_dirs') or [] module_build_dir = os.path.dirname(lib_file) if requiref90: self.mkpath(module_build_dir) if compiler.compiler_type == 'msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] objects = [] if c_sources: log.info("compiling C sources") objects = compiler.compile(c_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile(cxx_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) objects.extend(cxx_objects) if f_sources or fmodule_sources: extra_postargs = [] f_objects = [] if requiref90: if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options( module_dirs, module_build_dir) if fmodule_sources: log.info("compiling Fortran 90 module sources") f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if requiref90 and self._f_compiler.module_dir_switch is None: # move new compiled F90 module files to module_build_dir for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) else: f_objects = [] if f_objects and not fcompiler.can_ccompiler_link(compiler): # Default linker cannot link Fortran object files, and results # need to be wrapped later. Instead of creating a real static # library, just keep track of the object files. listfn = os.path.join(self.build_clib, lib_name + '.fobjects') with open(listfn, 'w') as f: f.write("\n".join(os.path.abspath(obj) for obj in f_objects)) listfn = os.path.join(self.build_clib, lib_name + '.cobjects') with open(listfn, 'w') as f: f.write("\n".join(os.path.abspath(obj) for obj in objects)) # create empty "library" file for dependency tracking lib_fname = os.path.join(self.build_clib, lib_name + compiler.static_lib_extension) with open(lib_fname, 'wb') as f: pass else: # assume that default linker is suitable for # linking Fortran object files objects.extend(f_objects) compiler.create_static_lib(objects, lib_name, output_dir=self.build_clib, debug=self.debug) # fix library dependencies clib_libraries = build_info.get('libraries', []) for lname, binfo in libraries: if lname in clib_libraries: clib_libraries.extend(binfo.get('libraries', [])) if clib_libraries: build_info['libraries'] = clib_libraries
def is_sequence_of_strings(seq): return is_sequence(seq) and all_strings(seq)