def _check_append_library(libraries, item): for libitem in libraries: if is_sequence(libitem): if is_sequence(item): if item[0]==libitem[0]: if item[1] is libitem[1]: return warnings.warn("[0] libraries list contains %r with" " different build_info" % (item[0],)) break else: if item==libitem[0]: warnings.warn("[1] libraries list contains %r with" " no build_info" % (item[0],)) break else: if is_sequence(item): if item[0]==libitem: warnings.warn("[2] libraries list contains %r with" " no build_info" % (item[0],)) break else: if item==libitem: return libraries.append(item)
def CCompiler_spawn(self, cmd, display=None): """ Execute a command in a sub-process. Parameters ---------- cmd : str The command to execute. display : str or sequence of str, optional The text to add to the log file kept by `numpy.distutils`. If not given, `display` is equal to `cmd`. Returns ------- None Raises ------ DistutilsExecError If the command failed, i.e. the exit status was not 0. """ if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) try: subprocess.check_output(cmd) except subprocess.CalledProcessError as exc: o = exc.output s = exc.returncode except OSError: # OSError doesn't have the same hooks for the exception # output, but exec_command() historically would use an # empty string for EnvironmentError (base class for # OSError) o = b'' # status previously used by exec_command() for parent # of OSError s = 127 else: # use a convenience return here so that any kind of # caught exception will execute the default code after the # try / except block, which handles various exceptions return None if is_sequence(cmd): cmd = ' '.join(list(cmd)) forward_bytes_to_stdout(o) if re.search(b'Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' else: msg = '' raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
def _exec_command(command, use_shell=None, use_tee = None, **env): """ Internal workhorse for exec_command(). """ if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' if os.name == 'posix' and use_shell: # On POSIX, subprocess always uses /bin/sh, override sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): command = [sh, '-c', ' '.join(command)] else: command = [sh, '-c', command] use_shell = False elif os.name == 'nt' and is_sequence(command): # On Windows, join the string for CreateProcess() ourselves as # subprocess does it a bit differently command = ' '.join(_quote_arg(arg) for arg in command) # Inherit environment by default env = env or None try: # universal_newlines is set to False so that communicate() # will return bytes. We need to decode the output ourselves # so that Python will not raise a UnicodeDecodeError when # it encounters an invalid character; rather, we simply replace it proc = subprocess.Popen(command, shell=use_shell, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=False) except EnvironmentError: # Return 127, as os.spawn*() and /bin/sh do return 127, '' text, err = proc.communicate() mylocale = locale.getpreferredencoding(False) if mylocale is None: mylocale = 'ascii' text = text.decode(mylocale, errors='replace') text = text.replace('\r\n', '\n') # Another historical oddity if text[-1:] == '\n': text = text[:-1] # stdio uses bytes in python 2, so to avoid issues, we simply # remove all non-ascii characters if sys.version_info < (3, 0): text = text.encode('ascii', errors='replace') if use_tee and text: print(text) return proc.returncode, text
def CCompiler_spawn(self, cmd, display=None): if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) if is_sequence(cmd) and os.name == 'nt': cmd = _nt_quote_args(list(cmd)) s,o = exec_command(cmd) if s: if is_sequence(cmd): cmd = ' '.join(list(cmd)) print o raise DistutilsExecError,\ 'Command "%s" failed with exit status %d' % (cmd, s)
def build_py_modules_sources(self): if not self.py_modules: return log.info('building py_modules sources') new_py_modules = [] for source in self.py_modules: if is_sequence(source) and len(source)==3: package, module_base, source = source if self.inplace: build_dir = self.get_package_dir(package) else: build_dir = os.path.join(self.build_src, os.path.join(*package.split('.'))) if hasattr(source, '__call__'): target = os.path.join(build_dir, module_base + '.py') source = source(target) if source is None: continue modules = [(package, module_base, source)] if package not in self.py_modules_dict: self.py_modules_dict[package] = [] self.py_modules_dict[package] += modules else: new_py_modules.append(source) self.py_modules[:] = new_py_modules
def template_sources(self, sources, extension): new_sources = [] if is_sequence(extension): depends = extension[1].get('depends') include_dirs = extension[1].get('include_dirs') else: depends = extension.depends include_dirs = extension.include_dirs for source in sources: (base, ext) = os.path.splitext(source) if ext == '.src': # Template file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) self.mkpath(target_dir) target_file = os.path.join(target_dir, os.path.basename(base)) if (self.force or newer_group([source] + depends, target_file)): if _f_pyf_ext_match(base): log.info("from_template:> %s" % (target_file)) outstr = process_f_file(source) else: log.info("conv_template:> %s" % (target_file)) outstr = process_c_file(source) with open(target_file, 'w') as fid: fid.write(outstr) if _header_ext_match(target_file): d = os.path.dirname(target_file) if d not in include_dirs: log.info(" adding '%s' to include_dirs." % (d)) include_dirs.append(d) new_sources.append(target_file) else: new_sources.append(source) return new_sources
def CCompiler_spawn(self, cmd, display=None): if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) s,o = exec_command(cmd) if s: if is_sequence(cmd): cmd = ' '.join(list(cmd)) print o if re.search('Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' else: msg = '' raise DistutilsExecError,\ 'Command "%s" failed with exit status %d%s' % (cmd, s, msg)
def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] for i in r: if is_sequence(i): lib_opts.extend(list(i)) else: lib_opts.append(i) return lib_opts
def CCompiler_spawn(self, cmd, display=None): """ Execute a command in a sub-process. Parameters ---------- cmd : str The command to execute. display : str or sequence of str, optional The text to add to the log file kept by `numpy.distutils`. If not given, `display` is equal to `cmd`. Returns ------- None Raises ------ DistutilsExecError If the command failed, i.e. the exit status was not 0. """ if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) s,o = exec_command(cmd) if s: if is_sequence(cmd): cmd = ' '.join(list(cmd)) try: print(o) except UnicodeError: # When installing through pip, `o` can contain non-ascii chars pass if re.search('Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' else: msg = '' raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
def _check_append_ext_library(libraries, lib_name, build_info): for item in libraries: if is_sequence(item): if item[0] == lib_name: if item[1] is build_info: return warnings.warn("[3] libraries list contains %r with" " different build_info" % (lib_name,)) break elif item == lib_name: warnings.warn("[4] libraries list contains %r with" " no build_info" % (lib_name,)) break libraries.append((lib_name, build_info))
def _exec_command(command, use_shell=None, use_tee = None, **env): """ Internal workhorse for exec_command(). """ if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' if os.name == 'posix' and use_shell: # On POSIX, subprocess always uses /bin/sh, override sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): command = [sh, '-c', ' '.join(command)] else: command = [sh, '-c', command] use_shell = False elif os.name == 'nt' and is_sequence(command): # On Windows, join the string for CreateProcess() ourselves as # subprocess does it a bit differently command = ' '.join(_quote_arg(arg) for arg in command) # Inherit environment by default env = env or None try: proc = subprocess.Popen(command, shell=use_shell, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) except EnvironmentError: # Return 127, as os.spawn*() and /bin/sh do return 127, '' text, err = proc.communicate() # Another historical oddity if text[-1:] == '\n': text = text[:-1] if use_tee and text: print(text) return proc.returncode, text
def _exec_command_posix( command, use_shell = None, use_tee = None, **env ): log.debug('_exec_command_posix(...)') if is_sequence(command): command_str = ' '.join(list(command)) else: command_str = command tmpfile = temp_file_name() stsfile = None if use_tee: stsfile = temp_file_name() filter = '' if use_tee == 2: filter = r'| tr -cd "\n" | tr "\n" "."; echo' command_posix = '( %s ; echo $? > %s ) 2>&1 | tee %s %s'\ % (command_str, stsfile, tmpfile, filter) else: stsfile = temp_file_name() command_posix = '( %s ; echo $? > %s ) > %s 2>&1'\ % (command_str, stsfile, tmpfile) #command_posix = '( %s ) > %s 2>&1' % (command_str,tmpfile) log.debug('Running os.system(%r)' % (command_posix)) status = os.system(command_posix) if use_tee: if status: # if command_tee fails then fall back to robust exec_command log.warn('_exec_command_posix failed (status=%s)' % status) return _exec_command(command, use_shell=use_shell, **env) if stsfile is not None: f = open_latin1(stsfile, 'r') status_text = f.read() status = int(status_text) f.close() os.remove(stsfile) f = open_latin1(tmpfile, 'r') text = f.read() f.close() os.remove(tmpfile) if text[-1:]=='\n': text = text[:-1] return status, text
def generate_sources(self, sources, extension): new_sources = [] func_sources = [] for source in sources: if is_string(source): new_sources.append(source) else: func_sources.append(source) if not func_sources: return new_sources if self.inplace and not is_sequence(extension): build_dir = self.ext_target_dir else: if is_sequence(extension): name = extension[0] # if 'include_dirs' not in extension[1]: # extension[1]['include_dirs'] = [] # incl_dirs = extension[1]['include_dirs'] else: name = extension.name # incl_dirs = extension.include_dirs #if self.build_src not in incl_dirs: # incl_dirs.append(self.build_src) build_dir = os.path.join(*([self.build_src]\ +name.split('.')[:-1])) self.mkpath(build_dir) for func in func_sources: source = func(extension, build_dir) if not source: continue if is_sequence(source): [log.info(" adding '%s' to sources." % (s,)) for s in source] new_sources.extend(source) else: log.info(" adding '%s' to sources." % (source,)) new_sources.append(source) return new_sources
def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): # the version of this function provided by CPython allows the following # to return lists, which are unpacked automatically: # - compiler.runtime_library_dir_option # our version extends the behavior to: # - compiler.library_dir_option # - compiler.library_option # - compiler.find_library_file r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] for i in r: if is_sequence(i): lib_opts.extend(list(i)) else: lib_opts.append(i) return lib_opts
def get_default_fcompiler(osname=None, platform=None, requiref90=None): """ Determine the default Fortran compiler to use for the given platform. """ if osname is None: osname = os.name if platform is None: platform = sys.platform matching_compilers = [] for pattern, compiler in _default_compilers: if re.match(pattern, platform) is not None or \ re.match(pattern, osname) is not None: if is_sequence(compiler): matching_compilers.extend(list(compiler)) else: matching_compilers.append(compiler) if not matching_compilers: matching_compilers.append('gnu') compiler = _find_existing_fcompiler(matching_compilers, osname=osname, platform=platform, requiref90=requiref90) if compiler is not None: return compiler return matching_compilers[0]
def f2py_sources(self, sources, extension): new_sources = [] f2py_sources = [] f_sources = [] f2py_targets = {} target_dirs = [] ext_name = extension.name.split('.')[-1] skip_f2py = 0 for source in sources: (base, ext) = os.path.splitext(source) if ext == '.pyf': # F2PY interface file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) if os.path.isfile(source): name = get_f2py_modulename(source) if name != ext_name: raise DistutilsSetupError( 'mismatch of extension names: %s ' 'provides %r but expected %r' % (source, name, ext_name)) target_file = os.path.join(target_dir, name + 'module.c') else: log.debug(' source %s does not exist: skipping f2py\'ing.' \ % (source)) name = ext_name skip_f2py = 1 target_file = os.path.join(target_dir, name + 'module.c') if not os.path.isfile(target_file): log.warn(' target %s does not exist:\n '\ 'Assuming %smodule.c was generated with '\ '"build_src --inplace" command.' \ % (target_file, name)) target_dir = os.path.dirname(base) target_file = os.path.join(target_dir, name + 'module.c') if not os.path.isfile(target_file): raise DistutilsSetupError("%r missing" % (target_file, )) log.info(' Yes! Using %r as up-to-date target.' \ % (target_file)) target_dirs.append(target_dir) f2py_sources.append(source) f2py_targets[source] = target_file new_sources.append(target_file) elif fortran_ext_match(ext): f_sources.append(source) else: new_sources.append(source) if not (f2py_sources or f_sources): return new_sources for d in target_dirs: self.mkpath(d) f2py_options = extension.f2py_options + self.f2py_opts if self.distribution.libraries: for name, build_info in self.distribution.libraries: if name in extension.libraries: f2py_options.extend(build_info.get('f2py_options', [])) log.info("f2py options: %s" % (f2py_options)) if f2py_sources: if len(f2py_sources) != 1: raise DistutilsSetupError( 'only one .pyf file is allowed per extension module but got'\ ' more: %r' % (f2py_sources,)) source = f2py_sources[0] target_file = f2py_targets[source] target_dir = os.path.dirname(target_file) or '.' depends = [source] + extension.depends if (self.force or newer_group(depends, target_file,'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) import numpy.f2py numpy.f2py.run_main(f2py_options + ['--build-dir', target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) else: #XXX TODO: --inplace support for sdist command if is_sequence(extension): name = extension[0] else: name = extension.name target_dir = os.path.join(*([self.build_src]\ +name.split('.')[:-1])) target_file = os.path.join(target_dir, ext_name + 'module.c') new_sources.append(target_file) depends = f_sources + extension.depends if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) import numpy.f2py numpy.f2py.run_main(f2py_options + ['--lower', '--build-dir',target_dir]+\ ['-m',ext_name]+f_sources) else: log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\ % (target_file)) if not os.path.isfile(target_file): raise DistutilsError("f2py target file %r not generated" % (target_file, )) target_c = os.path.join(self.build_src, 'fortranobject.c') target_h = os.path.join(self.build_src, 'fortranobject.h') log.info(" adding '%s' to sources." % (target_c)) new_sources.append(target_c) if self.build_src not in extension.include_dirs: log.info(" adding '%s' to include_dirs." \ % (self.build_src)) extension.include_dirs.append(self.build_src) if not skip_f2py: import numpy.f2py d = os.path.dirname(numpy.f2py.__file__) source_c = os.path.join(d, 'src', 'fortranobject.c') source_h = os.path.join(d, 'src', 'fortranobject.h') if newer(source_c, target_c) or newer(source_h, target_h): self.mkpath(os.path.dirname(target_c)) self.copy_file(source_c, target_c) self.copy_file(source_h, target_h) else: if not os.path.isfile(target_c): raise DistutilsSetupError("f2py target_c file %r not found" % (target_c, )) if not os.path.isfile(target_h): raise DistutilsSetupError("f2py target_h file %r not found" % (target_h, )) for name_ext in ['-f2pywrappers.f', '-f2pywrappers2.f90']: filename = os.path.join(target_dir, ext_name + name_ext) if os.path.isfile(filename): log.info(" adding '%s' to sources." % (filename)) f_sources.append(filename) return new_sources + f_sources
def _exec_command( command, use_shell=None, use_tee = None, **env ): log.debug('_exec_command(...)') if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get('SHELL','/bin/sh') if is_sequence(command): argv = [sh,'-c',' '.join(list(command))] else: argv = [sh,'-c',command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = splitcmdline(command) if hasattr(os,'spawnvpe'): spawn_command = os.spawnvpe else: spawn_command = os.spawnve argv[0] = find_executable(argv[0]) if not os.path.isfile(argv[0]): log.warn('Executable %s does not exist' % (argv[0])) if os.name in ['nt','dos']: # argv[0] might be internal command argv = [os.environ['COMSPEC'],'/C'] + argv using_command = 1 # sys.__std*__ is used instead of sys.std* because environments # like IDLE, PyCrust, etc overwrite sys.std* commands. so_fileno = sys.__stdout__.fileno() se_fileno = sys.__stderr__.fileno() so_flush = sys.__stdout__.flush se_flush = sys.__stderr__.flush so_dup = os.dup(so_fileno) se_dup = os.dup(se_fileno) outfile = tempfile.mktemp() fout = open(outfile,'w') if using_command: errfile = tempfile.mktemp() ferr = open(errfile,'w') log.debug('Running %s(%s,%r,%r,os.environ)' \ % (spawn_command.__name__,os.P_WAIT,argv[0],argv)) argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() os.dup2(fout.fileno(),so_fileno) if using_command: #XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(),se_fileno) else: os.dup2(fout.fileno(),se_fileno) try: status = spawn_command(os.P_WAIT,argv0,argv,os.environ) except OSError,errmess: status = 999 sys.stderr.write('%s: %s'%(errmess,argv[0]))
def flatten( df: pd.DataFrame, reset_index: bool = True, drop_levels: Union[Sequence[int], Sequence[str]] = (), ) -> pd.DataFrame: """ Convert N-dimensional DataFrame to a flat DataFrame :param df: N-dimensional DataFrame. :param reset_index: Convert index to column when df.index isn't RangeIndex :param drop_levels: index of level or names of level might be dropped if df is N-dimensional :return: a flat DataFrame Examples ----------- Convert DatetimeIndex into columns. >>> index = pd.to_datetime(["2021-01-01", "2021-01-02", "2021-01-03",]) >>> index.name = "__timestamp" >>> df = pd.DataFrame(index=index, data={"metric": [1, 2, 3]}) >>> df metric __timestamp 2021-01-01 1 2021-01-02 2 2021-01-03 3 >>> df = flatten(df) >>> df __timestamp metric 0 2021-01-01 1 1 2021-01-02 2 2 2021-01-03 3 Convert DatetimeIndex and MultipleIndex into columns >>> iterables = [["foo", "bar"], ["one", "two"]] >>> columns = pd.MultiIndex.from_product(iterables, names=["level1", "level2"]) >>> df = pd.DataFrame(index=index, columns=columns, data=1) >>> df level1 foo bar level2 one two one two __timestamp 2021-01-01 1 1 1 1 2021-01-02 1 1 1 1 2021-01-03 1 1 1 1 >>> flatten(df) __timestamp foo, one foo, two bar, one bar, two 0 2021-01-01 1 1 1 1 1 2021-01-02 1 1 1 1 2 2021-01-03 1 1 1 1 """ if _is_multi_index_on_columns(df): df.columns = df.columns.droplevel(drop_levels) _columns = [] for series in df.columns.to_flat_index(): _cells = [] for cell in series if is_sequence(series) else [series]: if pd.notnull(cell): # every cell should be converted to string _cells.append(str(cell)) _columns.append(FLAT_COLUMN_SEPARATOR.join(_cells)) df.columns = _columns if reset_index and not isinstance(df.index, pd.RangeIndex): df = df.reset_index(level=0) return df
def f2py_sources(self, sources, extension): new_sources = [] f2py_sources = [] f_sources = [] f2py_targets = {} target_dirs = [] ext_name = extension.name.split(".")[-1] skip_f2py = 0 for source in sources: (base, ext) = os.path.splitext(source) if ext == ".pyf": # F2PY interface file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) if os.path.isfile(source): name = get_f2py_modulename(source) if name != ext_name: raise DistutilsSetupError( "mismatch of extension names: %s " "provides %r but expected %r" % (source, name, ext_name) ) target_file = os.path.join(target_dir, name + "module.c") else: log.debug(" source %s does not exist: skipping f2py'ing." % (source)) name = ext_name skip_f2py = 1 target_file = os.path.join(target_dir, name + "module.c") if not os.path.isfile(target_file): log.warn( " target %s does not exist:\n " "Assuming %smodule.c was generated with " '"build_src --inplace" command.' % (target_file, name) ) target_dir = os.path.dirname(base) target_file = os.path.join(target_dir, name + "module.c") if not os.path.isfile(target_file): raise DistutilsSetupError("%r missing" % (target_file,)) log.info(" Yes! Using %r as up-to-date target." % (target_file)) target_dirs.append(target_dir) f2py_sources.append(source) f2py_targets[source] = target_file new_sources.append(target_file) elif fortran_ext_match(ext): f_sources.append(source) else: new_sources.append(source) if not (f2py_sources or f_sources): return new_sources for d in target_dirs: self.mkpath(d) f2py_options = extension.f2py_options + self.f2py_opts if self.distribution.libraries: for name, build_info in self.distribution.libraries: if name in extension.libraries: f2py_options.extend(build_info.get("f2py_options", [])) log.info("f2py options: %s" % (f2py_options)) if f2py_sources: if len(f2py_sources) != 1: raise DistutilsSetupError( "only one .pyf file is allowed per extension module but got" " more: %r" % (f2py_sources,) ) source = f2py_sources[0] target_file = f2py_targets[source] target_dir = os.path.dirname(target_file) or "." depends = [source] + extension.depends if (self.force or newer_group(depends, target_file, "newer")) and not skip_f2py: log.info("f2py: %s" % (source)) import numpy.f2py numpy.f2py.run_main(f2py_options + ["--build-dir", target_dir, source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) else: # XXX TODO: --inplace support for sdist command if is_sequence(extension): name = extension[0] else: name = extension.name target_dir = os.path.join(*([self.build_src] + name.split(".")[:-1])) target_file = os.path.join(target_dir, ext_name + "module.c") new_sources.append(target_file) depends = f_sources + extension.depends if (self.force or newer_group(depends, target_file, "newer")) and not skip_f2py: log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) import numpy.f2py numpy.f2py.run_main( f2py_options + ["--lower", "--build-dir", target_dir] + ["-m", ext_name] + f_sources ) else: log.debug(" skipping f2py fortran files for '%s' (up-to-date)" % (target_file)) if not os.path.isfile(target_file): raise DistutilsError("f2py target file %r not generated" % (target_file,)) target_c = os.path.join(self.build_src, "fortranobject.c") target_h = os.path.join(self.build_src, "fortranobject.h") log.info(" adding '%s' to sources." % (target_c)) new_sources.append(target_c) if self.build_src not in extension.include_dirs: log.info(" adding '%s' to include_dirs." % (self.build_src)) extension.include_dirs.append(self.build_src) if not skip_f2py: import numpy.f2py d = os.path.dirname(numpy.f2py.__file__) source_c = os.path.join(d, "src", "fortranobject.c") source_h = os.path.join(d, "src", "fortranobject.h") if newer(source_c, target_c) or newer(source_h, target_h): self.mkpath(os.path.dirname(target_c)) self.copy_file(source_c, target_c) self.copy_file(source_h, target_h) else: if not os.path.isfile(target_c): raise DistutilsSetupError("f2py target_c file %r not found" % (target_c,)) if not os.path.isfile(target_h): raise DistutilsSetupError("f2py target_h file %r not found" % (target_h,)) for name_ext in ["-f2pywrappers.f", "-f2pywrappers2.f90"]: filename = os.path.join(target_dir, ext_name + name_ext) if os.path.isfile(filename): log.info(" adding '%s' to sources." % (filename)) f_sources.append(filename) return new_sources + f_sources
if item == libitem[0]: warnings.warn("[1] libraries list contains %r with" " no build_info" % (item[0], )) break else: if is_sequence(item): if item[0] == libitem: warnings.warn("[2] libraries list contains %r with" " no build_info" % (item[0], )) break else: if item == libitem: return libraries.append(item) def _check_append_ext_library(libraries, (lib_name, build_info)): for item in libraries: if is_sequence(item): if item[0] == lib_name: if item[1] is build_info: return warnings.warn("[3] libraries list contains %r with" " different build_info" % (lib_name, )) break elif item == lib_name: warnings.warn("[4] libraries list contains %r with" " no build_info" % (lib_name, )) break libraries.append((lib_name, build_info))
def build_extension(self, ext): sources = ext.sources if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % ext.name) sources = list(sources) if not sources: return fullname = self.get_ext_fullname(ext.name) if self.inplace: modpath = fullname.split(".") package = ".".join(modpath[0:-1]) base = modpath[-1] build_py = self.get_finalized_command("build_py") package_dir = build_py.get_package_dir(package) ext_filename = os.path.join(package_dir, self.get_ext_filename(base)) else: ext_filename = os.path.join(self.build_lib, self.get_ext_filename(fullname)) depends = sources + ext.depends if not (self.force or newer_group(depends, ext_filename, "newer")): log.debug("skipping '%s' extension (up-to-date)", ext.name) return else: log.info("building '%s' extension", ext.name) extra_args = ext.extra_compile_args or [] macros = ext.define_macros[:] for undef in ext.undef_macros: macros.append((undef, )) c_sources, cxx_sources, f_sources, fmodule_sources = filter_sources( ext.sources) if self.compiler.compiler_type == "msvc": if cxx_sources: # Needed to compile kiva.agg._agg extension. extra_args.append("/Zm1000") # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] # Set Fortran/C++ compilers for compilation and linking. if ext.language == "f90": fcompiler = self._f90_compiler elif ext.language == "f77": fcompiler = self._f77_compiler else: # in case ext.language is c++, for instance fcompiler = self._f90_compiler or self._f77_compiler if fcompiler is not None: fcompiler.extra_f77_compile_args = (( ext.extra_f77_compile_args or []) if hasattr( ext, "extra_f77_compile_args") else []) fcompiler.extra_f90_compile_args = (( ext.extra_f90_compile_args or []) if hasattr( ext, "extra_f90_compile_args") else []) cxx_compiler = self._cxx_compiler # check for the availability of required compilers if cxx_sources and cxx_compiler is None: raise DistutilsError("extension %r has C++ sources" "but no C++ compiler found" % (ext.name)) if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("extension %r has Fortran sources " "but no Fortran compiler found" % (ext.name)) if ext.language in ["f77", "f90"] and fcompiler is None: self.warn("extension %r has Fortran libraries " "but no Fortran linker found, using default linker" % (ext.name)) if ext.language == "c++" and cxx_compiler is None: self.warn("extension %r has C++ libraries " "but no C++ linker found, using default linker" % (ext.name)) kws = {"depends": ext.depends} output_dir = self.build_temp include_dirs = ext.include_dirs + get_numpy_include_dirs() # filtering C dispatch-table sources when optimization is not disabled, # otherwise treated as normal sources. copt_c_sources = [] copt_baseline_flags = [] copt_macros = [] if not self.disable_optimization: bsrc_dir = self.get_finalized_command("build_src").build_src dispatch_hpath = os.path.join("numpy", "distutils", "include") dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath) include_dirs.append(dispatch_hpath) copt_build_src = None if self.inplace else bsrc_dir copt_c_sources = [ c_sources.pop(c_sources.index(src)) for src in c_sources[:] if src.endswith(".dispatch.c") ] copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) c_objects = [] if copt_c_sources: log.info("compiling C dispatch-able sources") c_objects += self.compiler_opt.try_dispatch( copt_c_sources, output_dir=output_dir, src_dir=copt_build_src, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) if c_sources: log.info("compiling C sources") c_objects += self.compiler.compile(c_sources, output_dir=output_dir, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args + copt_baseline_flags, **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args + copt_baseline_flags, **kws) extra_postargs = [] f_objects = [] if fmodule_sources: log.info("compiling Fortran 90 module sources") module_dirs = ext.module_dirs[:] module_build_dir = os.path.join( self.build_temp, os.path.dirname(self.get_ext_filename(fullname))) self.mkpath(module_build_dir) if fcompiler.module_dir_switch is None: existing_modules = glob("*.mod") extra_postargs += fcompiler.module_options(module_dirs, module_build_dir) f_objects += fcompiler.compile( fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends, ) if fcompiler.module_dir_switch is None: for f in glob("*.mod"): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn("failed to move %r to %r" % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile( f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends, ) if f_objects and not fcompiler.can_ccompiler_link(self.compiler): unlinkable_fobjects = f_objects objects = c_objects else: unlinkable_fobjects = [] objects = c_objects + f_objects if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] libraries = self.get_libraries(ext)[:] library_dirs = ext.library_dirs[:] linker = self.compiler.link_shared_object # Always use system linker when using MSVC compiler. if self.compiler.compiler_type in ("msvc", "intelw", "intelemw"): # expand libraries with fcompiler libraries as we are # not using fcompiler linker self._libs_with_msvc_and_fortran(fcompiler, libraries, library_dirs) elif ext.language in ["f77", "f90"] and fcompiler is not None: linker = fcompiler.link_shared_object if ext.language == "c++" and cxx_compiler is not None: linker = cxx_compiler.link_shared_object if fcompiler is not None: objects, libraries = self._process_unlinkable_fobjects( objects, libraries, fcompiler, library_dirs, unlinkable_fobjects) linker( objects, ext_filename, libraries=libraries, library_dirs=library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=ext.language, )
def _exec_command(command, use_shell=None, use_tee=None, **env): log.debug('_exec_command(...)') if use_shell is None: use_shell = os.name == 'posix' if use_tee is None: use_tee = os.name == 'posix' using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): argv = [sh, '-c', ' '.join(list(command))] else: argv = [sh, '-c', command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = shlex.split(command) if hasattr(os, 'spawnvpe'): spawn_command = os.spawnvpe else: spawn_command = os.spawnve argv[0] = find_executable(argv[0]) or argv[0] if not os.path.isfile(argv[0]): log.warn('Executable %s does not exist' % (argv[0])) if os.name in ['nt', 'dos']: # argv[0] might be internal command argv = [os.environ['COMSPEC'], '/C'] + argv using_command = 1 _so_has_fileno = _supports_fileno(sys.stdout) _se_has_fileno = _supports_fileno(sys.stderr) so_flush = sys.stdout.flush se_flush = sys.stderr.flush if _so_has_fileno: so_fileno = sys.stdout.fileno() so_dup = os.dup(so_fileno) if _se_has_fileno: se_fileno = sys.stderr.fileno() se_dup = os.dup(se_fileno) outfile = temp_file_name() fout = open(outfile, 'w') if using_command: errfile = temp_file_name() ferr = open(errfile, 'w') log.debug('Running %s(%s,%r,%r,os.environ)' \ % (spawn_command.__name__, os.P_WAIT, argv[0], argv)) if sys.version_info[0] >= 3 and os.name == 'nt': # Pre-encode os.environ, discarding un-encodable entries, # to avoid it failing during encoding as part of spawn. Failure # is possible if the environment contains entries that are not # encoded using the system codepage as windows expects. # # This is not necessary on unix, where os.environ is encoded # using the surrogateescape error handler and decoded using # it as part of spawn. encoded_environ = {} for k, v in os.environ.items(): try: encoded_environ[k.encode( sys.getfilesystemencoding())] = v.encode( sys.getfilesystemencoding()) except UnicodeEncodeError: log.debug("ignoring un-encodable env entry %s", k) else: encoded_environ = os.environ argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() if _so_has_fileno: os.dup2(fout.fileno(), so_fileno) if _se_has_fileno: if using_command: #XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(), se_fileno) else: os.dup2(fout.fileno(), se_fileno) try: status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ) except Exception: errmess = str(get_exception()) status = 999 sys.stderr.write('%s: %s' % (errmess, argv[0])) so_flush() se_flush() if _so_has_fileno: os.dup2(so_dup, so_fileno) os.close(so_dup) if _se_has_fileno: os.dup2(se_dup, se_fileno) os.close(se_dup) fout.close() fout = open_latin1(outfile, 'r') text = fout.read() fout.close() os.remove(outfile) if using_command: ferr.close() ferr = open_latin1(errfile, 'r') errmess = ferr.read() ferr.close() os.remove(errfile) if errmess and not status: # Not sure how to handle the case where errmess # contains only warning messages and that should # not be treated as errors. #status = 998 if text: text = text + '\n' #text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) text = text + errmess print(errmess) if text[-1:] == '\n': text = text[:-1] if status is None: status = 0 if use_tee: print(text) return status, text
def setup(**attr): if len(sys.argv) <= 1 and not attr.get("script_args", []): from interactive import interactive_sys_argv import atexit atexit.register(_exit_interactive_session) sys.argv[:] = interactive_sys_argv(sys.argv) if len(sys.argv) > 1: return setup(**attr) cmdclass = numpy_cmdclass.copy() new_attr = attr.copy() if "cmdclass" in new_attr: cmdclass.update(new_attr["cmdclass"]) new_attr["cmdclass"] = cmdclass if "configuration" in new_attr: # To avoid calling configuration if there are any errors # or help request in command in the line. configuration = new_attr.pop("configuration") old_dist = distutils.core._setup_distribution old_stop = distutils.core._setup_stop_after distutils.core._setup_distribution = None distutils.core._setup_stop_after = "commandline" try: dist = setup(**new_attr) finally: distutils.core._setup_distribution = old_dist distutils.core._setup_stop_after = old_stop if dist.help or not _command_line_ok(): # probably displayed help, skip running any commands return dist # create setup dictionary and append to new_attr config = configuration() if hasattr(config, "todict"): config = config.todict() _dict_append(new_attr, **config) # Move extension source libraries to libraries libraries = [] for ext in new_attr.get("ext_modules", []): new_libraries = [] for item in ext.libraries: if is_sequence(item): lib_name, build_info = item _check_append_ext_library(libraries, lib_name, build_info) new_libraries.append(lib_name) elif is_string(item): new_libraries.append(item) else: raise TypeError("invalid description of extension module " "library %r" % (item,)) ext.libraries = new_libraries if libraries: if "libraries" not in new_attr: new_attr["libraries"] = [] for item in libraries: _check_append_library(new_attr["libraries"], item) # sources in ext_modules or libraries may contain header files if ("ext_modules" in new_attr or "libraries" in new_attr) and "headers" not in new_attr: new_attr["headers"] = [] # Use our custom NumpyDistribution class instead of distutils' one new_attr["distclass"] = NumpyDistribution return old_setup(**new_attr)
def f2py_sources(self, sources, extension): new_sources = [] f2py_sources = [] f_sources = [] f2py_targets = {} target_dirs = [] ext_name = extension.name.split('.')[-1] skip_f2py = 0 for source in sources: (base, ext) = os.path.splitext(source) if ext == '.pyf': # F2PY interface file if self.inplace: target_dir = os.path.dirname(base) else: target_dir = appendpath(self.build_src, os.path.dirname(base)) if os.path.isfile(source): name = get_f2py_modulename(source) if name != ext_name: raise DistutilsSetupError('mismatch of extension names: %s ' 'provides %r but expected %r' % ( source, name, ext_name)) target_file = os.path.join(target_dir,name+'module.c') else: log.debug(' source %s does not exist: skipping f2py\'ing.' \ % (source)) name = ext_name skip_f2py = 1 target_file = os.path.join(target_dir,name+'module.c') if not os.path.isfile(target_file): log.warn(' target %s does not exist:\n '\ 'Assuming %smodule.c was generated with '\ '"build_src --inplace" command.' \ % (target_file, name)) target_dir = os.path.dirname(base) target_file = os.path.join(target_dir,name+'module.c') if not os.path.isfile(target_file): raise DistutilsSetupError("%r missing" % (target_file,)) log.info(' Yes! Using %r as up-to-date target.' \ % (target_file)) target_dirs.append(target_dir) f2py_sources.append(source) f2py_targets[source] = target_file new_sources.append(target_file) elif fortran_ext_match(ext): f_sources.append(source) else: new_sources.append(source) if not (f2py_sources or f_sources): return new_sources map(self.mkpath, target_dirs) f2py_options = extension.f2py_options + self.f2py_opts if self.distribution.libraries: for name,build_info in self.distribution.libraries: if name in extension.libraries: f2py_options.extend(build_info.get('f2py_options',[])) log.info("f2py options: %s" % (f2py_options)) if f2py_sources: if len(f2py_sources) != 1: raise DistutilsSetupError( 'only one .pyf file is allowed per extension module but got'\ ' more: %r' % (f2py_sources,)) source = f2py_sources[0] target_file = f2py_targets[source] target_dir = os.path.dirname(target_file) or '.' depends = [source] + extension.depends if (self.force or newer_group(depends, target_file,'newer')) \ and not skip_f2py: log.info("f2py: %s" % (source)) import numpy.f2py as f2py2e f2py2e.run_main(f2py_options + ['--build-dir',target_dir,source]) else: log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) else: #XXX TODO: --inplace support for sdist command if is_sequence(extension): name = extension[0] else: name = extension.name target_dir = os.path.join(*([self.build_src]\ +name.split('.')[:-1])) target_file = os.path.join(target_dir,ext_name + 'module.c') new_sources.append(target_file) depends = f_sources + extension.depends if (self.force or newer_group(depends, target_file, 'newer')) \ and not skip_f2py: import numpy.f2py as f2py2e log.info("f2py:> %s" % (target_file)) self.mkpath(target_dir) f2py2e.run_main(f2py_options + ['--lower', '--build-dir',target_dir]+\ ['-m',ext_name]+f_sources) else: log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\ % (target_file)) if not os.path.isfile(target_file): raise DistutilsError("f2py target file %r not generated" % (target_file,)) target_c = os.path.join(self.build_src,'fortranobject.c') target_h = os.path.join(self.build_src,'fortranobject.h') log.info(" adding '%s' to sources." % (target_c)) new_sources.append(target_c) if self.build_src not in extension.include_dirs: log.info(" adding '%s' to include_dirs." \ % (self.build_src)) extension.include_dirs.append(self.build_src) if not skip_f2py: import numpy.f2py as f2py2e d = os.path.dirname(f2py2e.__file__) source_c = os.path.join(d,'src','fortranobject.c') source_h = os.path.join(d,'src','fortranobject.h') if newer(source_c,target_c) or newer(source_h,target_h): self.mkpath(os.path.dirname(target_c)) self.copy_file(source_c,target_c) self.copy_file(source_h,target_h) else: if not os.path.isfile(target_c): raise DistutilsSetupError("f2py target_c file %r not found" % (target_c,)) if not os.path.isfile(target_h): raise DistutilsSetupError("f2py target_h file %r not found" % (target_h,)) for name_ext in ['-f2pywrappers.f','-f2pywrappers2.f90']: filename = os.path.join(target_dir,ext_name + name_ext) if os.path.isfile(filename): log.info(" adding '%s' to sources." % (filename)) f_sources.append(filename) return new_sources + f_sources
def _exec_command(command, use_shell=None, use_tee=None, **env): log.debug("_exec_command(...)") if use_shell is None: use_shell = os.name == "posix" if use_tee is None: use_tee = os.name == "posix" using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get("SHELL", "/bin/sh") if is_sequence(command): argv = [sh, "-c", " ".join(list(command))] else: argv = [sh, "-c", command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = shlex.split(command) # `spawn*p` family with path (vp, vpe, ...) are not available on windows. # Also prefer spawn{v,vp} in favor of spawn{ve,vpe} if no env # modification is actually requested as the *e* functions are not thread # safe on windows (https://bugs.python.org/issue6476) if hasattr(os, "spawnvpe"): spawn_command = os.spawnvpe if env else os.spawnvp else: spawn_command = os.spawnve if env else os.spawnv argv[0] = find_executable(argv[0]) or argv[0] if not os.path.isfile(argv[0]): log.warn("Executable %s does not exist" % (argv[0])) if os.name in ["nt", "dos"]: # argv[0] might be internal command argv = [os.environ["COMSPEC"], "/C"] + argv using_command = 1 _so_has_fileno = _supports_fileno(sys.stdout) _se_has_fileno = _supports_fileno(sys.stderr) so_flush = sys.stdout.flush se_flush = sys.stderr.flush if _so_has_fileno: so_fileno = sys.stdout.fileno() so_dup = os.dup(so_fileno) if _se_has_fileno: se_fileno = sys.stderr.fileno() se_dup = os.dup(se_fileno) outfile = temp_file_name() fout = open(outfile, "w") if using_command: errfile = temp_file_name() ferr = open(errfile, "w") log.debug("Running %s(%s,%r,%r,os.environ)" % (spawn_command.__name__, os.P_WAIT, argv[0], argv)) if env and sys.version_info[0] >= 3 and os.name == "nt": # Pre-encode os.environ, discarding un-encodable entries, # to avoid it failing during encoding as part of spawn. Failure # is possible if the environment contains entries that are not # encoded using the system codepage as windows expects. # # This is not necessary on unix, where os.environ is encoded # using the surrogateescape error handler and decoded using # it as part of spawn. encoded_environ = {} for k, v in os.environ.items(): try: encoded_environ[k.encode(sys.getfilesystemencoding())] = v.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: log.debug("ignoring un-encodable env entry %s", k) else: encoded_environ = os.environ argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() if _so_has_fileno: os.dup2(fout.fileno(), so_fileno) if _se_has_fileno: if using_command: # XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(), se_fileno) else: os.dup2(fout.fileno(), se_fileno) try: # Use spawnv in favor of spawnve, unless necessary if env: status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ) else: status = spawn_command(os.P_WAIT, argv0, argv) except Exception: errmess = str(get_exception()) status = 999 sys.stderr.write("%s: %s" % (errmess, argv[0])) so_flush() se_flush() if _so_has_fileno: os.dup2(so_dup, so_fileno) os.close(so_dup) if _se_has_fileno: os.dup2(se_dup, se_fileno) os.close(se_dup) fout.close() fout = open_latin1(outfile, "r") text = fout.read() fout.close() os.remove(outfile) if using_command: ferr.close() ferr = open_latin1(errfile, "r") errmess = ferr.read() ferr.close() os.remove(errfile) if errmess and not status: # Not sure how to handle the case where errmess # contains only warning messages and that should # not be treated as errors. # status = 998 if text: text = text + "\n" # text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) text = text + errmess print(errmess) if text[-1:] == "\n": text = text[:-1] if status is None: status = 0 if use_tee: print(text) return status, text
def setup(**attr): cmdclass = numpy_cmdclass.copy() new_attr = attr.copy() if 'cmdclass' in new_attr: cmdclass.update(new_attr['cmdclass']) new_attr['cmdclass'] = cmdclass if 'configuration' in new_attr: # To avoid calling configuration if there are any errors # or help request in command in the line. configuration = new_attr.pop('configuration') old_dist = distutils.core._setup_distribution old_stop = distutils.core._setup_stop_after distutils.core._setup_distribution = None distutils.core._setup_stop_after = "commandline" try: dist = setup(**new_attr) finally: distutils.core._setup_distribution = old_dist distutils.core._setup_stop_after = old_stop if dist.help or not _command_line_ok(): # probably displayed help, skip running any commands return dist # create setup dictionary and append to new_attr config = configuration() if hasattr(config, 'todict'): config = config.todict() _dict_append(new_attr, **config) # Move extension source libraries to libraries libraries = [] for ext in new_attr.get('ext_modules', []): new_libraries = [] for item in ext.libraries: if is_sequence(item): lib_name, build_info = item _check_append_ext_library(libraries, lib_name, build_info) new_libraries.append(lib_name) elif is_string(item): new_libraries.append(item) else: raise TypeError("invalid description of extension module " "library %r" % (item, )) ext.libraries = new_libraries if libraries: if 'libraries' not in new_attr: new_attr['libraries'] = [] for item in libraries: _check_append_library(new_attr['libraries'], item) # sources in ext_modules or libraries may contain header files if ('ext_modules' in new_attr or 'libraries' in new_attr) \ and 'headers' not in new_attr: new_attr['headers'] = [] # Use our custom NumpyDistribution class instead of distutils' one new_attr['distclass'] = NumpyDistribution return old_setup(**new_attr)
def is_sequence_of_strings(seq): return is_sequence(seq) and all_strings(seq)
def _fix_args(args, flag=1): if is_string(args): return args.replace('%', '%%') if flag and is_sequence(args): return tuple([_fix_args(a, flag=0) for a in args]) return args
def is_sequence_of_strings(seq): return is_sequence(seq) and all_strings(seq)
def build_extension(self, ext): sources = ext.sources if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % ext.name) sources = list(sources) if not sources: return fullname = self.get_ext_fullname(ext.name) if self.inplace: modpath = fullname.split('.') package = '.'.join(modpath[0:-1]) base = modpath[-1] build_py = self.get_finalized_command('build_py') package_dir = build_py.get_package_dir(package) ext_filename = os.path.join(package_dir, self.get_ext_filename(base)) else: ext_filename = os.path.join(self.build_lib, self.get_ext_filename(fullname)) depends = sources + ext.depends if not (self.force or newer_group(depends, ext_filename, 'newer')): log.debug("skipping '%s' extension (up-to-date)", ext.name) return else: log.info("building '%s' extension", ext.name) extra_args = ext.extra_compile_args or [] macros = ext.define_macros[:] for undef in ext.undef_macros: macros.append((undef, )) c_sources, cxx_sources, f_sources, fmodule_sources = \ filter_sources(ext.sources) if self.compiler.compiler_type == 'msvc': if cxx_sources: # Needed to compile kiva.agg._agg extension. extra_args.append('/Zm1000') # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] # Set Fortran/C++ compilers for compilation and linking. if ext.language == 'f90': fcompiler = self._f90_compiler elif ext.language == 'f77': fcompiler = self._f77_compiler else: # in case ext.language is c++, for instance fcompiler = self._f90_compiler or self._f77_compiler if fcompiler is not None: fcompiler.extra_f77_compile_args = ( ext.extra_f77_compile_args or []) if hasattr( ext, 'extra_f77_compile_args') else [] fcompiler.extra_f90_compile_args = ( ext.extra_f90_compile_args or []) if hasattr( ext, 'extra_f90_compile_args') else [] cxx_compiler = self._cxx_compiler # check for the availability of required compilers if cxx_sources and cxx_compiler is None: raise DistutilsError("extension %r has C++ sources" \ "but no C++ compiler found" % (ext.name)) if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("extension %r has Fortran sources " \ "but no Fortran compiler found" % (ext.name)) if ext.language in ['f77', 'f90'] and fcompiler is None: self.warn("extension %r has Fortran libraries " \ "but no Fortran linker found, using default linker" % (ext.name)) if ext.language == 'c++' and cxx_compiler is None: self.warn("extension %r has C++ libraries " \ "but no C++ linker found, using default linker" % (ext.name)) kws = {'depends': ext.depends} output_dir = self.build_temp include_dirs = ext.include_dirs + get_numpy_include_dirs() c_objects = [] if c_sources: log.info("compiling C sources") c_objects = self.compiler.compile(c_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) extra_postargs = [] f_objects = [] if fmodule_sources: log.info("compiling Fortran 90 module sources") module_dirs = ext.module_dirs[:] module_build_dir = os.path.join( self.build_temp, os.path.dirname(self.get_ext_filename(fullname))) self.mkpath(module_build_dir) if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options(module_dirs, module_build_dir) f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) if fcompiler.module_dir_switch is None: for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) objects = c_objects + f_objects if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] libraries = self.get_libraries(ext)[:] library_dirs = ext.library_dirs[:] linker = self.compiler.link_shared_object # Always use system linker when using MSVC compiler. if self.compiler.compiler_type == 'msvc': # expand libraries with fcompiler libraries as we are # not using fcompiler linker self._libs_with_msvc_and_fortran(fcompiler, libraries, library_dirs) elif ext.language in ['f77', 'f90'] and fcompiler is not None: linker = fcompiler.link_shared_object if ext.language == 'c++' and cxx_compiler is not None: linker = cxx_compiler.link_shared_object linker(objects, ext_filename, libraries=libraries, library_dirs=library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=ext.language)
def build_extension(self, ext): sources = ext.sources if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % ext.name) sources = list(sources) if not sources: return fullname = self.get_ext_fullname(ext.name) if self.inplace: modpath = fullname.split('.') package = '.'.join(modpath[0:-1]) base = modpath[-1] build_py = self.get_finalized_command('build_py') package_dir = build_py.get_package_dir(package) ext_filename = os.path.join(package_dir, self.get_ext_filename(base)) else: ext_filename = os.path.join(self.build_lib, self.get_ext_filename(fullname)) depends = sources + ext.depends if not (self.force or newer_group(depends, ext_filename, 'newer')): log.debug("skipping '%s' extension (up-to-date)", ext.name) return else: log.info("building '%s' extension", ext.name) extra_args = ext.extra_compile_args or [] macros = ext.define_macros[:] for undef in ext.undef_macros: macros.append((undef,)) c_sources, cxx_sources, f_sources, fmodule_sources = \ filter_sources(ext.sources) if self.compiler.compiler_type == 'msvc': if cxx_sources: # Needed to compile kiva.agg._agg extension. extra_args.append('/Zm1000') # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] # Set Fortran/C++ compilers for compilation and linking. if ext.language == 'f90': fcompiler = self._f90_compiler elif ext.language == 'f77': fcompiler = self._f77_compiler else: # in case ext.language is c++, for instance fcompiler = self._f90_compiler or self._f77_compiler if fcompiler is not None: fcompiler.extra_f77_compile_args = (ext.extra_f77_compile_args or []) if hasattr( ext, 'extra_f77_compile_args') else [] fcompiler.extra_f90_compile_args = (ext.extra_f90_compile_args or []) if hasattr( ext, 'extra_f90_compile_args') else [] cxx_compiler = self._cxx_compiler # check for the availability of required compilers if cxx_sources and cxx_compiler is None: raise DistutilsError("extension %r has C++ sources" "but no C++ compiler found" % (ext.name)) if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("extension %r has Fortran sources " "but no Fortran compiler found" % (ext.name)) if ext.language in ['f77', 'f90'] and fcompiler is None: self.warn("extension %r has Fortran libraries " "but no Fortran linker found, using default linker" % (ext.name)) if ext.language == 'c++' and cxx_compiler is None: self.warn("extension %r has C++ libraries " "but no C++ linker found, using default linker" % (ext.name)) kws = {'depends': ext.depends} output_dir = self.build_temp include_dirs = ext.include_dirs + get_numpy_include_dirs() c_objects = [] if c_sources: log.info("compiling C sources") c_objects = self.compiler.compile(c_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) if cxx_sources: log.info("compiling C++ sources") c_objects += cxx_compiler.compile(cxx_sources, output_dir=output_dir, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_args, **kws) extra_postargs = [] f_objects = [] if fmodule_sources: log.info("compiling Fortran 90 module sources") module_dirs = ext.module_dirs[:] module_build_dir = os.path.join( self.build_temp, os.path.dirname( self.get_ext_filename(fullname))) self.mkpath(module_build_dir) if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options( module_dirs, module_build_dir) f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) if fcompiler.module_dir_switch is None: for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, depends=ext.depends) if f_objects and not fcompiler.can_ccompiler_link(self.compiler): unlinkable_fobjects = f_objects objects = c_objects else: unlinkable_fobjects = [] objects = c_objects + f_objects if ext.extra_objects: objects.extend(ext.extra_objects) extra_args = ext.extra_link_args or [] libraries = self.get_libraries(ext)[:] library_dirs = ext.library_dirs[:] linker = self.compiler.link_shared_object # Always use system linker when using MSVC compiler. if self.compiler.compiler_type in ('msvc', 'intelw', 'intelemw'): # expand libraries with fcompiler libraries as we are # not using fcompiler linker self._libs_with_msvc_and_fortran( fcompiler, libraries, library_dirs) elif ext.language in ['f77', 'f90'] and fcompiler is not None: linker = fcompiler.link_shared_object if ext.language == 'c++' and cxx_compiler is not None: linker = cxx_compiler.link_shared_object if fcompiler is not None: objects, libraries = self._process_unlinkable_fobjects( objects, libraries, fcompiler, library_dirs, unlinkable_fobjects) linker(objects, ext_filename, libraries=libraries, library_dirs=library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=ext.language)
def _exec_command(command, use_shell=None, use_tee=None, **env): log.debug('_exec_command(...)') if use_shell is None: use_shell = os.name == 'posix' if use_tee is None: use_tee = os.name == 'posix' using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): argv = [sh, '-c', ' '.join(list(command))] else: argv = [sh, '-c', command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = shlex.split(command) if hasattr(os, 'spawnvpe'): spawn_command = os.spawnvpe else: spawn_command = os.spawnve argv[0] = find_executable(argv[0]) or argv[0] if not os.path.isfile(argv[0]): log.warn('Executable %s does not exist' % (argv[0])) if os.name in ['nt', 'dos']: # argv[0] might be internal command argv = [os.environ['COMSPEC'], '/C'] + argv using_command = 1 _so_has_fileno = _supports_fileno(sys.stdout) _se_has_fileno = _supports_fileno(sys.stderr) so_flush = sys.stdout.flush se_flush = sys.stderr.flush if _so_has_fileno: so_fileno = sys.stdout.fileno() so_dup = os.dup(so_fileno) if _se_has_fileno: se_fileno = sys.stderr.fileno() se_dup = os.dup(se_fileno) outfile = temp_file_name() fout = open(outfile, 'w') if using_command: errfile = temp_file_name() ferr = open(errfile, 'w') log.debug('Running %s(%s,%r,%r,os.environ)' \ % (spawn_command.__name__,os.P_WAIT,argv[0],argv)) argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() if _so_has_fileno: os.dup2(fout.fileno(), so_fileno) if _se_has_fileno: if using_command: #XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(), se_fileno) else: os.dup2(fout.fileno(), se_fileno) try: status = spawn_command(os.P_WAIT, argv0, argv, os.environ) except OSError: errmess = str(get_exception()) status = 999 sys.stderr.write('%s: %s' % (errmess, argv[0])) so_flush() se_flush() if _so_has_fileno: os.dup2(so_dup, so_fileno) if _se_has_fileno: os.dup2(se_dup, se_fileno) fout.close() fout = open_latin1(outfile, 'r') text = fout.read() fout.close() os.remove(outfile) if using_command: ferr.close() ferr = open_latin1(errfile, 'r') errmess = ferr.read() ferr.close() os.remove(errfile) if errmess and not status: # Not sure how to handle the case where errmess # contains only warning messages and that should # not be treated as errors. #status = 998 if text: text = text + '\n' #text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) text = text + errmess print(errmess) if text[-1:] == '\n': text = text[:-1] if status is None: status = 0 if use_tee: print(text) return status, text
def _load_file_and_cmd(self) -> None: """ Load settings from local file and arguments of the command line. """ def str_to_bool(arg_value: str) -> bool: """ Used to handle boolean settings. If not the 'bool' type convert all not empty string as true. :param arg_value: The boolean value as a string. :return: The value parsed as a string. """ if isinstance(arg_value, bool): return arg_value if arg_value.lower() in {'false', 'f', '0', 'no', 'n'}: return False elif arg_value.lower() in {'true', 't', '1', 'yes', 'y'}: return True raise argparse.ArgumentTypeError( f'{arg_value} is not a valid boolean value') def type_mapping(arg_value): if type(arg_value) == bool: return str_to_bool if is_sequence(arg_value): if len(arg_value) == 0: return str else: return type_mapping(arg_value[0]) # Default same as current value return type(arg_value) p = configargparse.get_argument_parser( default_config_files=['./settings.yaml']) # Spacial argument p.add_argument('-s', '--settings', required=False, is_config_file=True, help='path to custom configuration file') # Create argument for each attribute of this class for name, value in asdict(self).items(): p.add_argument(f'--{name.replace("_", "-")}', f'--{name}', dest=name, required=False, action='append' if is_sequence(value) else 'store', type=type_mapping(value)) # Load arguments form file, environment and command line to override the defaults for name, value in vars(p.parse_args()).items(): if name == 'settings': continue if value is not None: # Directly set the value to bypass the "__setattr__" function self.__dict__[name] = value self.validate()
def CCompiler_spawn(self, cmd, display=None): """ Execute a command in a sub-process. Parameters ---------- cmd : str The command to execute. display : str or sequence of str, optional The text to add to the log file kept by `numpy.distutils`. If not given, `display` is equal to `cmd`. Returns ------- None Raises ------ DistutilsExecError If the command failed, i.e. the exit status was not 0. """ if display is None: display = cmd if is_sequence(display): display = ' '.join(list(display)) log.info(display) try: if self.verbose: subprocess.check_output(cmd) else: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: o = exc.output s = exc.returncode except OSError: # OSError doesn't have the same hooks for the exception # output, but exec_command() historically would use an # empty string for EnvironmentError (base class for # OSError) o = b'' # status previously used by exec_command() for parent # of OSError s = 127 else: # use a convenience return here so that any kind of # caught exception will execute the default code after the # try / except block, which handles various exceptions return None if is_sequence(cmd): cmd = ' '.join(list(cmd)) if self.verbose: forward_bytes_to_stdout(o) if re.search(b'Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' else: msg = '' raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
def setup(**attr): cmdclass = numpy_cmdclass.copy() new_attr = attr.copy() if 'cmdclass' in new_attr: cmdclass.update(new_attr['cmdclass']) new_attr['cmdclass'] = cmdclass if 'configuration' in new_attr: # To avoid calling configuration if there are any errors # or help request in command in the line. configuration = new_attr.pop('configuration') old_dist = distutils.core._setup_distribution old_stop = distutils.core._setup_stop_after distutils.core._setup_distribution = None distutils.core._setup_stop_after = "commandline" try: dist = setup(**new_attr) finally: distutils.core._setup_distribution = old_dist distutils.core._setup_stop_after = old_stop if dist.help or not _command_line_ok(): # probably displayed help, skip running any commands return dist # create setup dictionary and append to new_attr config = configuration() if hasattr(config,'todict'): config = config.todict() _dict_append(new_attr, **config) # Move extension source libraries to libraries libraries = [] for ext in new_attr.get('ext_modules',[]): new_libraries = [] for item in ext.libraries: if is_sequence(item): lib_name, build_info = item _check_append_ext_library(libraries, lib_name, build_info) new_libraries.append(lib_name) elif is_string(item): new_libraries.append(item) else: raise TypeError("invalid description of extension module " "library %r" % (item,)) ext.libraries = new_libraries if libraries: if 'libraries' not in new_attr: new_attr['libraries'] = [] for item in libraries: _check_append_library(new_attr['libraries'], item) # sources in ext_modules or libraries may contain header files if ('ext_modules' in new_attr or 'libraries' in new_attr) \ and 'headers' not in new_attr: new_attr['headers'] = [] # Use our custom NumpyDistribution class instead of distutils' one new_attr['distclass'] = NumpyDistribution return old_setup(**new_attr)
def _exec_command( command, use_shell=None, use_tee = None, **env ): log.debug('_exec_command(...)') if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): argv = [sh, '-c', ' '.join(list(command))] else: argv = [sh, '-c', command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = shlex.split(command) if hasattr(os, 'spawnvpe'): spawn_command = os.spawnvpe else: spawn_command = os.spawnve argv[0] = find_executable(argv[0]) or argv[0] if not os.path.isfile(argv[0]): log.warn('Executable %s does not exist' % (argv[0])) if os.name in ['nt', 'dos']: # argv[0] might be internal command argv = [os.environ['COMSPEC'], '/C'] + argv using_command = 1 _so_has_fileno = _supports_fileno(sys.stdout) _se_has_fileno = _supports_fileno(sys.stderr) so_flush = sys.stdout.flush se_flush = sys.stderr.flush if _so_has_fileno: so_fileno = sys.stdout.fileno() so_dup = os.dup(so_fileno) if _se_has_fileno: se_fileno = sys.stderr.fileno() se_dup = os.dup(se_fileno) outfile = temp_file_name() fout = open(outfile, 'w') if using_command: errfile = temp_file_name() ferr = open(errfile, 'w') log.debug('Running %s(%s,%r,%r,os.environ)' \ % (spawn_command.__name__, os.P_WAIT, argv[0], argv)) if sys.version_info[0] >= 3 and os.name == 'nt': # Pre-encode os.environ, discarding un-encodable entries, # to avoid it failing during encoding as part of spawn. Failure # is possible if the environment contains entries that are not # encoded using the system codepage as windows expects. # # This is not necessary on unix, where os.environ is encoded # using the surrogateescape error handler and decoded using # it as part of spawn. encoded_environ = {} for k, v in os.environ.items(): try: encoded_environ[k.encode(sys.getfilesystemencoding())] = v.encode( sys.getfilesystemencoding()) except UnicodeEncodeError: log.debug("ignoring un-encodable env entry %s", k) else: encoded_environ = os.environ argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() if _so_has_fileno: os.dup2(fout.fileno(), so_fileno) if _se_has_fileno: if using_command: #XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(), se_fileno) else: os.dup2(fout.fileno(), se_fileno) try: status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ) except Exception: errmess = str(get_exception()) status = 999 sys.stderr.write('%s: %s'%(errmess, argv[0])) so_flush() se_flush() if _so_has_fileno: os.dup2(so_dup, so_fileno) os.close(so_dup) if _se_has_fileno: os.dup2(se_dup, se_fileno) os.close(se_dup) fout.close() fout = open_latin1(outfile, 'r') text = fout.read() fout.close() os.remove(outfile) if using_command: ferr.close() ferr = open_latin1(errfile, 'r') errmess = ferr.read() ferr.close() os.remove(errfile) if errmess and not status: # Not sure how to handle the case where errmess # contains only warning messages and that should # not be treated as errors. #status = 998 if text: text = text + '\n' #text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) text = text + errmess print (errmess) if text[-1:]=='\n': text = text[:-1] if status is None: status = 0 if use_tee: print (text) return status, text
def build_a_library(self, build_info, lib_name, libraries): # default compilers compiler = self.compiler fcompiler = self._f_compiler sources = build_info.get("sources") if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) sources = list(sources) c_sources, cxx_sources, f_sources, fmodule_sources = filter_sources( sources) requiref90 = not not fmodule_sources or build_info.get( "language", "c") == "f90" # save source type information so that build_ext can use it. source_languages = [] if c_sources: source_languages.append("c") if cxx_sources: source_languages.append("c++") if requiref90: source_languages.append("f90") elif f_sources: source_languages.append("f77") build_info["source_languages"] = source_languages lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) depends = sources + build_info.get("depends", []) if not (self.force or newer_group(depends, lib_file, "newer")): log.debug("skipping '%s' library (up-to-date)", lib_name) return else: log.info("building '%s' library", lib_name) config_fc = build_info.get("config_fc", {}) if fcompiler is not None and config_fc: log.info("using additional config_fc from setup script " "for fortran compiler: %s" % (config_fc, )) from numpy.distutils.fcompiler import new_fcompiler fcompiler = new_fcompiler( compiler=fcompiler.compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force, requiref90=requiref90, c_compiler=self.compiler, ) if fcompiler is not None: dist = self.distribution base_config_fc = dist.get_option_dict("config_fc").copy() base_config_fc.update(config_fc) fcompiler.customize(base_config_fc) # check availability of Fortran compilers if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("library %s has Fortran sources" " but no Fortran compiler found" % (lib_name)) if fcompiler is not None: fcompiler.extra_f77_compile_args = ( build_info.get("extra_f77_compile_args") or []) fcompiler.extra_f90_compile_args = ( build_info.get("extra_f90_compile_args") or []) macros = build_info.get("macros") if macros is None: macros = [] include_dirs = build_info.get("include_dirs") if include_dirs is None: include_dirs = [] extra_postargs = build_info.get("extra_compiler_args") or [] include_dirs.extend(get_numpy_include_dirs()) # where compiled F90 module files are: module_dirs = build_info.get("module_dirs") or [] module_build_dir = os.path.dirname(lib_file) if requiref90: self.mkpath(module_build_dir) if compiler.compiler_type == "msvc": # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] # filtering C dispatch-table sources when optimization is not disabled, # otherwise treated as normal sources. copt_c_sources = [] copt_baseline_flags = [] copt_macros = [] if not self.disable_optimization: bsrc_dir = self.get_finalized_command("build_src").build_src dispatch_hpath = os.path.join("numpy", "distutils", "include") dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath) include_dirs.append(dispatch_hpath) copt_build_src = None if self.inplace else bsrc_dir copt_c_sources = [ c_sources.pop(c_sources.index(src)) for src in c_sources[:] if src.endswith(".dispatch.c") ] copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) objects = [] if copt_c_sources: log.info("compiling C dispatch-able sources") objects += self.compiler_opt.try_dispatch( copt_c_sources, output_dir=self.build_temp, src_dir=copt_build_src, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, ) if c_sources: log.info("compiling C sources") objects += compiler.compile( c_sources, output_dir=self.build_temp, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs + copt_baseline_flags, ) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile( cxx_sources, output_dir=self.build_temp, macros=macros + copt_macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs + copt_baseline_flags, ) objects.extend(cxx_objects) if f_sources or fmodule_sources: extra_postargs = [] f_objects = [] if requiref90: if fcompiler.module_dir_switch is None: existing_modules = glob("*.mod") extra_postargs += fcompiler.module_options( module_dirs, module_build_dir) if fmodule_sources: log.info("compiling Fortran 90 module sources") f_objects += fcompiler.compile( fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, ) if requiref90 and self._f_compiler.module_dir_switch is None: # move new compiled F90 module files to module_build_dir for f in glob("*.mod"): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn("failed to move %r to %r" % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile( f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs, ) else: f_objects = [] if f_objects and not fcompiler.can_ccompiler_link(compiler): # Default linker cannot link Fortran object files, and results # need to be wrapped later. Instead of creating a real static # library, just keep track of the object files. listfn = os.path.join(self.build_clib, lib_name + ".fobjects") with open(listfn, "w") as f: f.write("\n".join(os.path.abspath(obj) for obj in f_objects)) listfn = os.path.join(self.build_clib, lib_name + ".cobjects") with open(listfn, "w") as f: f.write("\n".join(os.path.abspath(obj) for obj in objects)) # create empty "library" file for dependency tracking lib_fname = os.path.join(self.build_clib, lib_name + compiler.static_lib_extension) with open(lib_fname, "wb") as f: pass else: # assume that default linker is suitable for # linking Fortran object files objects.extend(f_objects) compiler.create_static_lib(objects, lib_name, output_dir=self.build_clib, debug=self.debug) # fix library dependencies clib_libraries = build_info.get("libraries", []) for lname, binfo in libraries: if lname in clib_libraries: clib_libraries.extend(binfo.get("libraries", [])) if clib_libraries: build_info["libraries"] = clib_libraries
def _fix_args(args,flag=1): if is_string(args): return args.replace('%','%%') if flag and is_sequence(args): return tuple([_fix_args(a,flag=0) for a in args]) return args
def _exec_command( command, use_shell=None, use_tee = None, **env ): log.debug('_exec_command(...)') if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' using_command = 0 if use_shell: # We use shell (unless use_shell==0) so that wildcards can be # used. sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): argv = [sh, '-c', ' '.join(list(command))] else: argv = [sh, '-c', command] else: # On NT, DOS we avoid using command.com as it's exit status is # not related to the exit status of a command. if is_sequence(command): argv = command[:] else: argv = shlex.split(command) if hasattr(os, 'spawnvpe'): spawn_command = os.spawnvpe else: spawn_command = os.spawnve argv[0] = find_executable(argv[0]) or argv[0] if not os.path.isfile(argv[0]): log.warn('Executable %s does not exist' % (argv[0])) if os.name in ['nt', 'dos']: # argv[0] might be internal command argv = [os.environ['COMSPEC'], '/C'] + argv using_command = 1 _so_has_fileno = _supports_fileno(sys.stdout) _se_has_fileno = _supports_fileno(sys.stderr) so_flush = sys.stdout.flush se_flush = sys.stderr.flush if _so_has_fileno: so_fileno = sys.stdout.fileno() so_dup = os.dup(so_fileno) if _se_has_fileno: se_fileno = sys.stderr.fileno() se_dup = os.dup(se_fileno) outfile = temp_file_name() fout = open(outfile, 'w') if using_command: errfile = temp_file_name() ferr = open(errfile, 'w') log.debug('Running %s(%s,%r,%r,os.environ)' \ % (spawn_command.__name__, os.P_WAIT, argv[0], argv)) argv0 = argv[0] if not using_command: argv[0] = quote_arg(argv0) so_flush() se_flush() if _so_has_fileno: os.dup2(fout.fileno(), so_fileno) if _se_has_fileno: if using_command: #XXX: disabled for now as it does not work from cmd under win32. # Tests fail on msys os.dup2(ferr.fileno(), se_fileno) else: os.dup2(fout.fileno(), se_fileno) try: status = spawn_command(os.P_WAIT, argv0, argv, os.environ) except OSError: errmess = str(get_exception()) status = 999 sys.stderr.write('%s: %s'%(errmess, argv[0])) so_flush() se_flush() if _so_has_fileno: os.dup2(so_dup, so_fileno) if _se_has_fileno: os.dup2(se_dup, se_fileno) fout.close() fout = open_latin1(outfile, 'r') text = fout.read() fout.close() os.remove(outfile) if using_command: ferr.close() ferr = open_latin1(errfile, 'r') errmess = ferr.read() ferr.close() os.remove(errfile) if errmess and not status: # Not sure how to handle the case where errmess # contains only warning messages and that should # not be treated as errors. #status = 998 if text: text = text + '\n' #text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) text = text + errmess print (errmess) if text[-1:]=='\n': text = text[:-1] if status is None: status = 0 if use_tee: print (text) return status, text
else: if item==libitem[0]: warnings.warn("[1] libraries list contains %r with" " no build_info" % (item[0],)) break else: if is_sequence(item): if item[0]==libitem: warnings.warn("[2] libraries list contains %r with" " no build_info" % (item[0],)) break else: if item==libitem: return libraries.append(item) def _check_append_ext_library(libraries, (lib_name,build_info)): for item in libraries: if is_sequence(item): if item[0]==lib_name: if item[1] is build_info: return warnings.warn("[3] libraries list contains %r with" " different build_info" % (lib_name,)) break elif item==lib_name: warnings.warn("[4] libraries list contains %r with" " no build_info" % (lib_name,)) break libraries.append((lib_name,build_info))
def build_a_library(self, build_info, lib_name, libraries): # default compilers compiler = self.compiler fcompiler = self._f_compiler sources = build_info.get('sources') if sources is None or not is_sequence(sources): raise DistutilsSetupError(("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) sources = list(sources) c_sources, cxx_sources, f_sources, fmodule_sources \ = filter_sources(sources) requiref90 = not not fmodule_sources or \ build_info.get('language', 'c')=='f90' # save source type information so that build_ext can use it. source_languages = [] if c_sources: source_languages.append('c') if cxx_sources: source_languages.append('c++') if requiref90: source_languages.append('f90') elif f_sources: source_languages.append('f77') build_info['source_languages'] = source_languages lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) depends = sources + build_info.get('depends', []) if not (self.force or newer_group(depends, lib_file, 'newer')): log.debug("skipping '%s' library (up-to-date)", lib_name) return else: log.info("building '%s' library", lib_name) config_fc = build_info.get('config_fc', {}) if fcompiler is not None and config_fc: log.info('using additional config_fc from setup script '\ 'for fortran compiler: %s' \ % (config_fc,)) from numpy.distutils.fcompiler import new_fcompiler fcompiler = new_fcompiler(compiler=fcompiler.compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force, requiref90=requiref90, c_compiler=self.compiler) if fcompiler is not None: dist = self.distribution base_config_fc = dist.get_option_dict('config_fc').copy() base_config_fc.update(config_fc) fcompiler.customize(base_config_fc) # check availability of Fortran compilers if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("library %s has Fortran sources"\ " but no Fortran compiler found" % (lib_name)) if fcompiler is not None: fcompiler.extra_f77_compile_args = build_info.get('extra_f77_compile_args') or [] fcompiler.extra_f90_compile_args = build_info.get('extra_f90_compile_args') or [] macros = build_info.get('macros') include_dirs = build_info.get('include_dirs') if include_dirs is None: include_dirs = [] extra_postargs = build_info.get('extra_compiler_args') or [] include_dirs.extend(get_numpy_include_dirs()) # where compiled F90 module files are: module_dirs = build_info.get('module_dirs') or [] module_build_dir = os.path.dirname(lib_file) if requiref90: self.mkpath(module_build_dir) if compiler.compiler_type=='msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] objects = [] if c_sources: log.info("compiling C sources") objects = compiler.compile(c_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile(cxx_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) objects.extend(cxx_objects) if f_sources or fmodule_sources: extra_postargs = [] f_objects = [] if requiref90: if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options(\ module_dirs, module_build_dir) if fmodule_sources: log.info("compiling Fortran 90 module sources") f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if requiref90 and self._f_compiler.module_dir_switch is None: # move new compiled F90 module files to module_build_dir for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f)==os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' \ % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) else: f_objects = [] objects.extend(f_objects) # assume that default linker is suitable for # linking Fortran object files compiler.create_static_lib(objects, lib_name, output_dir=self.build_clib, debug=self.debug) # fix library dependencies clib_libraries = build_info.get('libraries', []) for lname, binfo in libraries: if lname in clib_libraries: clib_libraries.extend(binfo[1].get('libraries', [])) if clib_libraries: build_info['libraries'] = clib_libraries
def build_a_library(self, build_info, lib_name, libraries): # default compilers compiler = self.compiler fcompiler = self._f_compiler sources = build_info.get('sources') if sources is None or not is_sequence(sources): raise DistutilsSetupError( ("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) sources = list(sources) c_sources, cxx_sources, f_sources, fmodule_sources \ = filter_sources(sources) requiref90 = not not fmodule_sources or \ build_info.get('language', 'c') == 'f90' # save source type information so that build_ext can use it. source_languages = [] if c_sources: source_languages.append('c') if cxx_sources: source_languages.append('c++') if requiref90: source_languages.append('f90') elif f_sources: source_languages.append('f77') build_info['source_languages'] = source_languages lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) depends = sources + build_info.get('depends', []) if not (self.force or newer_group(depends, lib_file, 'newer')): log.debug("skipping '%s' library (up-to-date)", lib_name) return else: log.info("building '%s' library", lib_name) config_fc = build_info.get('config_fc', {}) if fcompiler is not None and config_fc: log.info('using additional config_fc from setup script ' 'for fortran compiler: %s' % (config_fc, )) from numpy.distutils.fcompiler import new_fcompiler fcompiler = new_fcompiler(compiler=fcompiler.compiler_type, verbose=self.verbose, dry_run=self.dry_run, force=self.force, requiref90=requiref90, c_compiler=self.compiler) if fcompiler is not None: dist = self.distribution base_config_fc = dist.get_option_dict('config_fc').copy() base_config_fc.update(config_fc) fcompiler.customize(base_config_fc) # check availability of Fortran compilers if (f_sources or fmodule_sources) and fcompiler is None: raise DistutilsError("library %s has Fortran sources" " but no Fortran compiler found" % (lib_name)) if fcompiler is not None: fcompiler.extra_f77_compile_args = build_info.get( 'extra_f77_compile_args') or [] fcompiler.extra_f90_compile_args = build_info.get( 'extra_f90_compile_args') or [] macros = build_info.get('macros') include_dirs = build_info.get('include_dirs') if include_dirs is None: include_dirs = [] extra_postargs = build_info.get('extra_compiler_args') or [] include_dirs.extend(get_numpy_include_dirs()) # where compiled F90 module files are: module_dirs = build_info.get('module_dirs') or [] module_build_dir = os.path.dirname(lib_file) if requiref90: self.mkpath(module_build_dir) if compiler.compiler_type == 'msvc': # this hack works around the msvc compiler attributes # problem, msvc uses its own convention :( c_sources += cxx_sources cxx_sources = [] objects = [] if c_sources: log.info("compiling C sources") objects = compiler.compile(c_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if cxx_sources: log.info("compiling C++ sources") cxx_compiler = compiler.cxx_compiler() cxx_objects = cxx_compiler.compile(cxx_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) objects.extend(cxx_objects) if f_sources or fmodule_sources: extra_postargs = [] f_objects = [] if requiref90: if fcompiler.module_dir_switch is None: existing_modules = glob('*.mod') extra_postargs += fcompiler.module_options( module_dirs, module_build_dir) if fmodule_sources: log.info("compiling Fortran 90 module sources") f_objects += fcompiler.compile(fmodule_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) if requiref90 and self._f_compiler.module_dir_switch is None: # move new compiled F90 module files to module_build_dir for f in glob('*.mod'): if f in existing_modules: continue t = os.path.join(module_build_dir, f) if os.path.abspath(f) == os.path.abspath(t): continue if os.path.isfile(t): os.remove(t) try: self.move_file(f, module_build_dir) except DistutilsFileError: log.warn('failed to move %r to %r' % (f, module_build_dir)) if f_sources: log.info("compiling Fortran sources") f_objects += fcompiler.compile(f_sources, output_dir=self.build_temp, macros=macros, include_dirs=include_dirs, debug=self.debug, extra_postargs=extra_postargs) else: f_objects = [] if f_objects and not fcompiler.can_ccompiler_link(compiler): # Default linker cannot link Fortran object files, and results # need to be wrapped later. Instead of creating a real static # library, just keep track of the object files. listfn = os.path.join(self.build_clib, lib_name + '.fobjects') with open(listfn, 'w') as f: f.write("\n".join(os.path.abspath(obj) for obj in f_objects)) listfn = os.path.join(self.build_clib, lib_name + '.cobjects') with open(listfn, 'w') as f: f.write("\n".join(os.path.abspath(obj) for obj in objects)) # create empty "library" file for dependency tracking lib_fname = os.path.join(self.build_clib, lib_name + compiler.static_lib_extension) with open(lib_fname, 'wb') as f: pass else: # assume that default linker is suitable for # linking Fortran object files objects.extend(f_objects) compiler.create_static_lib(objects, lib_name, output_dir=self.build_clib, debug=self.debug) # fix library dependencies clib_libraries = build_info.get('libraries', []) for lname, binfo in libraries: if lname in clib_libraries: clib_libraries.extend(binfo.get('libraries', [])) if clib_libraries: build_info['libraries'] = clib_libraries