def _check_append_library(libraries, item):
for libitem in libraries:
if is_sequence(libitem):
if is_sequence(item):
if item[0]==libitem[0]:
if item[1] is libitem[1]:
return
warnings.warn("[0] libraries list contains %r with"
" different build_info" % (item[0],))
break
else:
if item==libitem[0]:
warnings.warn("[1] libraries list contains %r with"
" no build_info" % (item[0],))
break
else:
if is_sequence(item):
if item[0]==libitem:
warnings.warn("[2] libraries list contains %r with"
" no build_info" % (item[0],))
break
else:
if item==libitem:
return
libraries.append(item)
def CCompiler_spawn(self, cmd, display=None):
"""
Execute a command in a sub-process.
Parameters
----------
cmd : str
The command to execute.
display : str or sequence of str, optional
The text to add to the log file kept by `numpy.distutils`.
If not given, `display` is equal to `cmd`.
Returns
-------
None
Raises
------
DistutilsExecError
If the command failed, i.e. the exit status was not 0.
"""
if display is None:
display = cmd
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
try:
subprocess.check_output(cmd)
except subprocess.CalledProcessError as exc:
o = exc.output
s = exc.returncode
except OSError:
# OSError doesn't have the same hooks for the exception
# output, but exec_command() historically would use an
# empty string for EnvironmentError (base class for
# OSError)
o = b''
# status previously used by exec_command() for parent
# of OSError
s = 127
else:
# use a convenience return here so that any kind of
# caught exception will execute the default code after the
# try / except block, which handles various exceptions
return None
if is_sequence(cmd):
cmd = ' '.join(list(cmd))
forward_bytes_to_stdout(o)
if re.search(b'Too many open files', o):
msg = '\nTry rerunning setup command until build succeeds.'
else:
msg = ''
raise DistutilsExecError('Command "%s" failed with exit status %d%s' %
(cmd, s, msg))
def _exec_command(command, use_shell=None, use_tee = None, **env):
"""
Internal workhorse for exec_command().
"""
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
if os.name == 'posix' and use_shell:
# On POSIX, subprocess always uses /bin/sh, override
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
command = [sh, '-c', ' '.join(command)]
else:
command = [sh, '-c', command]
use_shell = False
elif os.name == 'nt' and is_sequence(command):
# On Windows, join the string for CreateProcess() ourselves as
# subprocess does it a bit differently
command = ' '.join(_quote_arg(arg) for arg in command)
# Inherit environment by default
env = env or None
try:
# universal_newlines is set to False so that communicate()
# will return bytes. We need to decode the output ourselves
# so that Python will not raise a UnicodeDecodeError when
# it encounters an invalid character; rather, we simply replace it
proc = subprocess.Popen(command, shell=use_shell, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=False)
except EnvironmentError:
# Return 127, as os.spawn*() and /bin/sh do
return 127, ''
text, err = proc.communicate()
mylocale = locale.getpreferredencoding(False)
if mylocale is None:
mylocale = 'ascii'
text = text.decode(mylocale, errors='replace')
text = text.replace('\r\n', '\n')
# Another historical oddity
if text[-1:] == '\n':
text = text[:-1]
# stdio uses bytes in python 2, so to avoid issues, we simply
# remove all non-ascii characters
if sys.version_info < (3, 0):
text = text.encode('ascii', errors='replace')
if use_tee and text:
print(text)
return proc.returncode, text
def CCompiler_spawn(self, cmd, display=None):
if display is None:
display = cmd
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
if is_sequence(cmd) and os.name == 'nt':
cmd = _nt_quote_args(list(cmd))
s,o = exec_command(cmd)
if s:
if is_sequence(cmd):
cmd = ' '.join(list(cmd))
print o
raise DistutilsExecError,\
'Command "%s" failed with exit status %d' % (cmd, s)
def build_py_modules_sources(self):
if not self.py_modules:
return
log.info('building py_modules sources')
new_py_modules = []
for source in self.py_modules:
if is_sequence(source) and len(source)==3:
package, module_base, source = source
if self.inplace:
build_dir = self.get_package_dir(package)
else:
build_dir = os.path.join(self.build_src,
os.path.join(*package.split('.')))
if hasattr(source, '__call__'):
target = os.path.join(build_dir, module_base + '.py')
source = source(target)
if source is None:
continue
modules = [(package, module_base, source)]
if package not in self.py_modules_dict:
self.py_modules_dict[package] = []
self.py_modules_dict[package] += modules
else:
new_py_modules.append(source)
self.py_modules[:] = new_py_modules
def template_sources(self, sources, extension):
new_sources = []
if is_sequence(extension):
depends = extension[1].get('depends')
include_dirs = extension[1].get('include_dirs')
else:
depends = extension.depends
include_dirs = extension.include_dirs
for source in sources:
(base, ext) = os.path.splitext(source)
if ext == '.src': # Template file
if self.inplace:
target_dir = os.path.dirname(base)
else:
target_dir = appendpath(self.build_src, os.path.dirname(base))
self.mkpath(target_dir)
target_file = os.path.join(target_dir, os.path.basename(base))
if (self.force or newer_group([source] + depends, target_file)):
if _f_pyf_ext_match(base):
log.info("from_template:> %s" % (target_file))
outstr = process_f_file(source)
else:
log.info("conv_template:> %s" % (target_file))
outstr = process_c_file(source)
with open(target_file, 'w') as fid:
fid.write(outstr)
if _header_ext_match(target_file):
d = os.path.dirname(target_file)
if d not in include_dirs:
log.info(" adding '%s' to include_dirs." % (d))
include_dirs.append(d)
new_sources.append(target_file)
else:
new_sources.append(source)
return new_sources
def CCompiler_spawn(self, cmd, display=None):
if display is None:
display = cmd
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
s,o = exec_command(cmd)
if s:
if is_sequence(cmd):
cmd = ' '.join(list(cmd))
print o
if re.search('Too many open files', o):
msg = '\nTry rerunning setup command until build succeeds.'
else:
msg = ''
raise DistutilsExecError,\
'Command "%s" failed with exit status %d%s' % (cmd, s, msg)
def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries):
r = _distutils_gen_lib_options(compiler, library_dirs,
runtime_library_dirs, libraries)
lib_opts = []
for i in r:
if is_sequence(i):
lib_opts.extend(list(i))
else:
lib_opts.append(i)
return lib_opts
def CCompiler_spawn(self, cmd, display=None):
"""
Execute a command in a sub-process.
Parameters
----------
cmd : str
The command to execute.
display : str or sequence of str, optional
The text to add to the log file kept by `numpy.distutils`.
If not given, `display` is equal to `cmd`.
Returns
-------
None
Raises
------
DistutilsExecError
If the command failed, i.e. the exit status was not 0.
"""
if display is None:
display = cmd
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
s,o = exec_command(cmd)
if s:
if is_sequence(cmd):
cmd = ' '.join(list(cmd))
try:
print(o)
except UnicodeError:
# When installing through pip, `o` can contain non-ascii chars
pass
if re.search('Too many open files', o):
msg = '\nTry rerunning setup command until build succeeds.'
else:
msg = ''
raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
def _check_append_ext_library(libraries, lib_name, build_info):
for item in libraries:
if is_sequence(item):
if item[0] == lib_name:
if item[1] is build_info:
return
warnings.warn("[3] libraries list contains %r with" " different build_info" % (lib_name,))
break
elif item == lib_name:
warnings.warn("[4] libraries list contains %r with" " no build_info" % (lib_name,))
break
libraries.append((lib_name, build_info))
def _exec_command(command, use_shell=None, use_tee = None, **env):
"""
Internal workhorse for exec_command().
"""
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
if os.name == 'posix' and use_shell:
# On POSIX, subprocess always uses /bin/sh, override
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
command = [sh, '-c', ' '.join(command)]
else:
command = [sh, '-c', command]
use_shell = False
elif os.name == 'nt' and is_sequence(command):
# On Windows, join the string for CreateProcess() ourselves as
# subprocess does it a bit differently
command = ' '.join(_quote_arg(arg) for arg in command)
# Inherit environment by default
env = env or None
try:
proc = subprocess.Popen(command, shell=use_shell, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
except EnvironmentError:
# Return 127, as os.spawn*() and /bin/sh do
return 127, ''
text, err = proc.communicate()
# Another historical oddity
if text[-1:] == '\n':
text = text[:-1]
if use_tee and text:
print(text)
return proc.returncode, text
def _exec_command_posix( command,
use_shell = None,
use_tee = None,
**env ):
log.debug('_exec_command_posix(...)')
if is_sequence(command):
command_str = ' '.join(list(command))
else:
command_str = command
tmpfile = temp_file_name()
stsfile = None
if use_tee:
stsfile = temp_file_name()
filter = ''
if use_tee == 2:
filter = r'| tr -cd "\n" | tr "\n" "."; echo'
command_posix = '( %s ; echo $? > %s ) 2>&1 | tee %s %s'\
% (command_str, stsfile, tmpfile, filter)
else:
stsfile = temp_file_name()
command_posix = '( %s ; echo $? > %s ) > %s 2>&1'\
% (command_str, stsfile, tmpfile)
#command_posix = '( %s ) > %s 2>&1' % (command_str,tmpfile)
log.debug('Running os.system(%r)' % (command_posix))
status = os.system(command_posix)
if use_tee:
if status:
# if command_tee fails then fall back to robust exec_command
log.warn('_exec_command_posix failed (status=%s)' % status)
return _exec_command(command, use_shell=use_shell, **env)
if stsfile is not None:
f = open_latin1(stsfile, 'r')
status_text = f.read()
status = int(status_text)
f.close()
os.remove(stsfile)
f = open_latin1(tmpfile, 'r')
text = f.read()
f.close()
os.remove(tmpfile)
if text[-1:]=='\n':
text = text[:-1]
return status, text
def generate_sources(self, sources, extension):
new_sources = []
func_sources = []
for source in sources:
if is_string(source):
new_sources.append(source)
else:
func_sources.append(source)
if not func_sources:
return new_sources
if self.inplace and not is_sequence(extension):
build_dir = self.ext_target_dir
else:
if is_sequence(extension):
name = extension[0]
# if 'include_dirs' not in extension[1]:
# extension[1]['include_dirs'] = []
# incl_dirs = extension[1]['include_dirs']
else:
name = extension.name
# incl_dirs = extension.include_dirs
#if self.build_src not in incl_dirs:
# incl_dirs.append(self.build_src)
build_dir = os.path.join(*([self.build_src]\
+name.split('.')[:-1]))
self.mkpath(build_dir)
for func in func_sources:
source = func(extension, build_dir)
if not source:
continue
if is_sequence(source):
[log.info(" adding '%s' to sources." % (s,)) for s in source]
new_sources.extend(source)
else:
log.info(" adding '%s' to sources." % (source,))
new_sources.append(source)
return new_sources
def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries):
# the version of this function provided by CPython allows the following
# to return lists, which are unpacked automatically:
# - compiler.runtime_library_dir_option
# our version extends the behavior to:
# - compiler.library_dir_option
# - compiler.library_option
# - compiler.find_library_file
r = _distutils_gen_lib_options(compiler, library_dirs,
runtime_library_dirs, libraries)
lib_opts = []
for i in r:
if is_sequence(i):
lib_opts.extend(list(i))
else:
lib_opts.append(i)
return lib_opts
def get_default_fcompiler(osname=None, platform=None, requiref90=None):
""" Determine the default Fortran compiler to use for the given platform. """
if osname is None:
osname = os.name
if platform is None:
platform = sys.platform
matching_compilers = []
for pattern, compiler in _default_compilers:
if re.match(pattern, platform) is not None or \
re.match(pattern, osname) is not None:
if is_sequence(compiler):
matching_compilers.extend(list(compiler))
else:
matching_compilers.append(compiler)
if not matching_compilers:
matching_compilers.append('gnu')
compiler = _find_existing_fcompiler(matching_compilers,
osname=osname,
platform=platform,
requiref90=requiref90)
if compiler is not None:
return compiler
return matching_compilers[0]
def f2py_sources(self, sources, extension):
new_sources = []
f2py_sources = []
f_sources = []
f2py_targets = {}
target_dirs = []
ext_name = extension.name.split('.')[-1]
skip_f2py = 0
for source in sources:
(base, ext) = os.path.splitext(source)
if ext == '.pyf': # F2PY interface file
if self.inplace:
target_dir = os.path.dirname(base)
else:
target_dir = appendpath(self.build_src,
os.path.dirname(base))
if os.path.isfile(source):
name = get_f2py_modulename(source)
if name != ext_name:
raise DistutilsSetupError(
'mismatch of extension names: %s '
'provides %r but expected %r' %
(source, name, ext_name))
target_file = os.path.join(target_dir, name + 'module.c')
else:
log.debug(' source %s does not exist: skipping f2py\'ing.' \
% (source))
name = ext_name
skip_f2py = 1
target_file = os.path.join(target_dir, name + 'module.c')
if not os.path.isfile(target_file):
log.warn(' target %s does not exist:\n '\
'Assuming %smodule.c was generated with '\
'"build_src --inplace" command.' \
% (target_file, name))
target_dir = os.path.dirname(base)
target_file = os.path.join(target_dir,
name + 'module.c')
if not os.path.isfile(target_file):
raise DistutilsSetupError("%r missing" %
(target_file, ))
log.info(' Yes! Using %r as up-to-date target.' \
% (target_file))
target_dirs.append(target_dir)
f2py_sources.append(source)
f2py_targets[source] = target_file
new_sources.append(target_file)
elif fortran_ext_match(ext):
f_sources.append(source)
else:
new_sources.append(source)
if not (f2py_sources or f_sources):
return new_sources
for d in target_dirs:
self.mkpath(d)
f2py_options = extension.f2py_options + self.f2py_opts
if self.distribution.libraries:
for name, build_info in self.distribution.libraries:
if name in extension.libraries:
f2py_options.extend(build_info.get('f2py_options', []))
log.info("f2py options: %s" % (f2py_options))
if f2py_sources:
if len(f2py_sources) != 1:
raise DistutilsSetupError(
'only one .pyf file is allowed per extension module but got'\
' more: %r' % (f2py_sources,))
source = f2py_sources[0]
target_file = f2py_targets[source]
target_dir = os.path.dirname(target_file) or '.'
depends = [source] + extension.depends
if (self.force or newer_group(depends, target_file,'newer')) \
and not skip_f2py:
log.info("f2py: %s" % (source))
import numpy.f2py
numpy.f2py.run_main(f2py_options +
['--build-dir', target_dir, source])
else:
log.debug(" skipping '%s' f2py interface (up-to-date)" %
(source))
else:
#XXX TODO: --inplace support for sdist command
if is_sequence(extension):
name = extension[0]
else:
name = extension.name
target_dir = os.path.join(*([self.build_src]\
+name.split('.')[:-1]))
target_file = os.path.join(target_dir, ext_name + 'module.c')
new_sources.append(target_file)
depends = f_sources + extension.depends
if (self.force or newer_group(depends, target_file, 'newer')) \
and not skip_f2py:
log.info("f2py:> %s" % (target_file))
self.mkpath(target_dir)
import numpy.f2py
numpy.f2py.run_main(f2py_options + ['--lower',
'--build-dir',target_dir]+\
['-m',ext_name]+f_sources)
else:
log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\
% (target_file))
if not os.path.isfile(target_file):
raise DistutilsError("f2py target file %r not generated" %
(target_file, ))
target_c = os.path.join(self.build_src, 'fortranobject.c')
target_h = os.path.join(self.build_src, 'fortranobject.h')
log.info(" adding '%s' to sources." % (target_c))
new_sources.append(target_c)
if self.build_src not in extension.include_dirs:
log.info(" adding '%s' to include_dirs." \
% (self.build_src))
extension.include_dirs.append(self.build_src)
if not skip_f2py:
import numpy.f2py
d = os.path.dirname(numpy.f2py.__file__)
source_c = os.path.join(d, 'src', 'fortranobject.c')
source_h = os.path.join(d, 'src', 'fortranobject.h')
if newer(source_c, target_c) or newer(source_h, target_h):
self.mkpath(os.path.dirname(target_c))
self.copy_file(source_c, target_c)
self.copy_file(source_h, target_h)
else:
if not os.path.isfile(target_c):
raise DistutilsSetupError("f2py target_c file %r not found" %
(target_c, ))
if not os.path.isfile(target_h):
raise DistutilsSetupError("f2py target_h file %r not found" %
(target_h, ))
for name_ext in ['-f2pywrappers.f', '-f2pywrappers2.f90']:
filename = os.path.join(target_dir, ext_name + name_ext)
if os.path.isfile(filename):
log.info(" adding '%s' to sources." % (filename))
f_sources.append(filename)
return new_sources + f_sources
def _exec_command( command, use_shell=None, use_tee = None, **env ):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL','/bin/sh')
if is_sequence(command):
argv = [sh,'-c',' '.join(list(command))]
else:
argv = [sh,'-c',command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = splitcmdline(command)
if hasattr(os,'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0])
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt','dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'],'/C'] + argv
using_command = 1
# sys.__std*__ is used instead of sys.std* because environments
# like IDLE, PyCrust, etc overwrite sys.std* commands.
so_fileno = sys.__stdout__.fileno()
se_fileno = sys.__stderr__.fileno()
so_flush = sys.__stdout__.flush
se_flush = sys.__stderr__.flush
so_dup = os.dup(so_fileno)
se_dup = os.dup(se_fileno)
outfile = tempfile.mktemp()
fout = open(outfile,'w')
if using_command:
errfile = tempfile.mktemp()
ferr = open(errfile,'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__,os.P_WAIT,argv[0],argv))
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
os.dup2(fout.fileno(),so_fileno)
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(),se_fileno)
else:
os.dup2(fout.fileno(),se_fileno)
try:
status = spawn_command(os.P_WAIT,argv0,argv,os.environ)
except OSError,errmess:
status = 999
sys.stderr.write('%s: %s'%(errmess,argv[0]))
def flatten(
df: pd.DataFrame,
reset_index: bool = True,
drop_levels: Union[Sequence[int], Sequence[str]] = (),
) -> pd.DataFrame:
"""
Convert N-dimensional DataFrame to a flat DataFrame
:param df: N-dimensional DataFrame.
:param reset_index: Convert index to column when df.index isn't RangeIndex
:param drop_levels: index of level or names of level might be dropped
if df is N-dimensional
:return: a flat DataFrame
Examples
-----------
Convert DatetimeIndex into columns.
>>> index = pd.to_datetime(["2021-01-01", "2021-01-02", "2021-01-03",])
>>> index.name = "__timestamp"
>>> df = pd.DataFrame(index=index, data={"metric": [1, 2, 3]})
>>> df
metric
__timestamp
2021-01-01 1
2021-01-02 2
2021-01-03 3
>>> df = flatten(df)
>>> df
__timestamp metric
0 2021-01-01 1
1 2021-01-02 2
2 2021-01-03 3
Convert DatetimeIndex and MultipleIndex into columns
>>> iterables = [["foo", "bar"], ["one", "two"]]
>>> columns = pd.MultiIndex.from_product(iterables, names=["level1", "level2"])
>>> df = pd.DataFrame(index=index, columns=columns, data=1)
>>> df
level1 foo bar
level2 one two one two
__timestamp
2021-01-01 1 1 1 1
2021-01-02 1 1 1 1
2021-01-03 1 1 1 1
>>> flatten(df)
__timestamp foo, one foo, two bar, one bar, two
0 2021-01-01 1 1 1 1
1 2021-01-02 1 1 1 1
2 2021-01-03 1 1 1 1
"""
if _is_multi_index_on_columns(df):
df.columns = df.columns.droplevel(drop_levels)
_columns = []
for series in df.columns.to_flat_index():
_cells = []
for cell in series if is_sequence(series) else [series]:
if pd.notnull(cell):
# every cell should be converted to string
_cells.append(str(cell))
_columns.append(FLAT_COLUMN_SEPARATOR.join(_cells))
df.columns = _columns
if reset_index and not isinstance(df.index, pd.RangeIndex):
df = df.reset_index(level=0)
return df
def f2py_sources(self, sources, extension):
new_sources = []
f2py_sources = []
f_sources = []
f2py_targets = {}
target_dirs = []
ext_name = extension.name.split(".")[-1]
skip_f2py = 0
for source in sources:
(base, ext) = os.path.splitext(source)
if ext == ".pyf": # F2PY interface file
if self.inplace:
target_dir = os.path.dirname(base)
else:
target_dir = appendpath(self.build_src, os.path.dirname(base))
if os.path.isfile(source):
name = get_f2py_modulename(source)
if name != ext_name:
raise DistutilsSetupError(
"mismatch of extension names: %s " "provides %r but expected %r" % (source, name, ext_name)
)
target_file = os.path.join(target_dir, name + "module.c")
else:
log.debug(" source %s does not exist: skipping f2py'ing." % (source))
name = ext_name
skip_f2py = 1
target_file = os.path.join(target_dir, name + "module.c")
if not os.path.isfile(target_file):
log.warn(
" target %s does not exist:\n "
"Assuming %smodule.c was generated with "
'"build_src --inplace" command.' % (target_file, name)
)
target_dir = os.path.dirname(base)
target_file = os.path.join(target_dir, name + "module.c")
if not os.path.isfile(target_file):
raise DistutilsSetupError("%r missing" % (target_file,))
log.info(" Yes! Using %r as up-to-date target." % (target_file))
target_dirs.append(target_dir)
f2py_sources.append(source)
f2py_targets[source] = target_file
new_sources.append(target_file)
elif fortran_ext_match(ext):
f_sources.append(source)
else:
new_sources.append(source)
if not (f2py_sources or f_sources):
return new_sources
for d in target_dirs:
self.mkpath(d)
f2py_options = extension.f2py_options + self.f2py_opts
if self.distribution.libraries:
for name, build_info in self.distribution.libraries:
if name in extension.libraries:
f2py_options.extend(build_info.get("f2py_options", []))
log.info("f2py options: %s" % (f2py_options))
if f2py_sources:
if len(f2py_sources) != 1:
raise DistutilsSetupError(
"only one .pyf file is allowed per extension module but got" " more: %r" % (f2py_sources,)
)
source = f2py_sources[0]
target_file = f2py_targets[source]
target_dir = os.path.dirname(target_file) or "."
depends = [source] + extension.depends
if (self.force or newer_group(depends, target_file, "newer")) and not skip_f2py:
log.info("f2py: %s" % (source))
import numpy.f2py
numpy.f2py.run_main(f2py_options + ["--build-dir", target_dir, source])
else:
log.debug(" skipping '%s' f2py interface (up-to-date)" % (source))
else:
# XXX TODO: --inplace support for sdist command
if is_sequence(extension):
name = extension[0]
else:
name = extension.name
target_dir = os.path.join(*([self.build_src] + name.split(".")[:-1]))
target_file = os.path.join(target_dir, ext_name + "module.c")
new_sources.append(target_file)
depends = f_sources + extension.depends
if (self.force or newer_group(depends, target_file, "newer")) and not skip_f2py:
log.info("f2py:> %s" % (target_file))
self.mkpath(target_dir)
import numpy.f2py
numpy.f2py.run_main(
f2py_options + ["--lower", "--build-dir", target_dir] + ["-m", ext_name] + f_sources
)
else:
log.debug(" skipping f2py fortran files for '%s' (up-to-date)" % (target_file))
if not os.path.isfile(target_file):
raise DistutilsError("f2py target file %r not generated" % (target_file,))
target_c = os.path.join(self.build_src, "fortranobject.c")
target_h = os.path.join(self.build_src, "fortranobject.h")
log.info(" adding '%s' to sources." % (target_c))
new_sources.append(target_c)
if self.build_src not in extension.include_dirs:
log.info(" adding '%s' to include_dirs." % (self.build_src))
extension.include_dirs.append(self.build_src)
if not skip_f2py:
import numpy.f2py
d = os.path.dirname(numpy.f2py.__file__)
source_c = os.path.join(d, "src", "fortranobject.c")
source_h = os.path.join(d, "src", "fortranobject.h")
if newer(source_c, target_c) or newer(source_h, target_h):
self.mkpath(os.path.dirname(target_c))
self.copy_file(source_c, target_c)
self.copy_file(source_h, target_h)
else:
if not os.path.isfile(target_c):
raise DistutilsSetupError("f2py target_c file %r not found" % (target_c,))
if not os.path.isfile(target_h):
raise DistutilsSetupError("f2py target_h file %r not found" % (target_h,))
for name_ext in ["-f2pywrappers.f", "-f2pywrappers2.f90"]:
filename = os.path.join(target_dir, ext_name + name_ext)
if os.path.isfile(filename):
log.info(" adding '%s' to sources." % (filename))
f_sources.append(filename)
return new_sources + f_sources
if item == libitem[0]:
warnings.warn("[1] libraries list contains %r with"
" no build_info" % (item[0], ))
break
else:
if is_sequence(item):
if item[0] == libitem:
warnings.warn("[2] libraries list contains %r with"
" no build_info" % (item[0], ))
break
else:
if item == libitem:
return
libraries.append(item)
def _check_append_ext_library(libraries, (lib_name, build_info)):
for item in libraries:
if is_sequence(item):
if item[0] == lib_name:
if item[1] is build_info:
return
warnings.warn("[3] libraries list contains %r with"
" different build_info" % (lib_name, ))
break
elif item == lib_name:
warnings.warn("[4] libraries list contains %r with"
" no build_info" % (lib_name, ))
break
libraries.append((lib_name, build_info))
def build_extension(self, ext):
sources = ext.sources
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(
("in 'ext_modules' option (extension '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % ext.name)
sources = list(sources)
if not sources:
return
fullname = self.get_ext_fullname(ext.name)
if self.inplace:
modpath = fullname.split(".")
package = ".".join(modpath[0:-1])
base = modpath[-1]
build_py = self.get_finalized_command("build_py")
package_dir = build_py.get_package_dir(package)
ext_filename = os.path.join(package_dir,
self.get_ext_filename(base))
else:
ext_filename = os.path.join(self.build_lib,
self.get_ext_filename(fullname))
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_filename, "newer")):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef, ))
c_sources, cxx_sources, f_sources, fmodule_sources = filter_sources(
ext.sources)
if self.compiler.compiler_type == "msvc":
if cxx_sources:
# Needed to compile kiva.agg._agg extension.
extra_args.append("/Zm1000")
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
# Set Fortran/C++ compilers for compilation and linking.
if ext.language == "f90":
fcompiler = self._f90_compiler
elif ext.language == "f77":
fcompiler = self._f77_compiler
else: # in case ext.language is c++, for instance
fcompiler = self._f90_compiler or self._f77_compiler
if fcompiler is not None:
fcompiler.extra_f77_compile_args = ((
ext.extra_f77_compile_args or []) if hasattr(
ext, "extra_f77_compile_args") else [])
fcompiler.extra_f90_compile_args = ((
ext.extra_f90_compile_args or []) if hasattr(
ext, "extra_f90_compile_args") else [])
cxx_compiler = self._cxx_compiler
# check for the availability of required compilers
if cxx_sources and cxx_compiler is None:
raise DistutilsError("extension %r has C++ sources"
"but no C++ compiler found" % (ext.name))
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("extension %r has Fortran sources "
"but no Fortran compiler found" % (ext.name))
if ext.language in ["f77", "f90"] and fcompiler is None:
self.warn("extension %r has Fortran libraries "
"but no Fortran linker found, using default linker" %
(ext.name))
if ext.language == "c++" and cxx_compiler is None:
self.warn("extension %r has C++ libraries "
"but no C++ linker found, using default linker" %
(ext.name))
kws = {"depends": ext.depends}
output_dir = self.build_temp
include_dirs = ext.include_dirs + get_numpy_include_dirs()
# filtering C dispatch-table sources when optimization is not disabled,
# otherwise treated as normal sources.
copt_c_sources = []
copt_baseline_flags = []
copt_macros = []
if not self.disable_optimization:
bsrc_dir = self.get_finalized_command("build_src").build_src
dispatch_hpath = os.path.join("numpy", "distutils", "include")
dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath)
include_dirs.append(dispatch_hpath)
copt_build_src = None if self.inplace else bsrc_dir
copt_c_sources = [
c_sources.pop(c_sources.index(src)) for src in c_sources[:]
if src.endswith(".dispatch.c")
]
copt_baseline_flags = self.compiler_opt.cpu_baseline_flags()
else:
copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1))
c_objects = []
if copt_c_sources:
log.info("compiling C dispatch-able sources")
c_objects += self.compiler_opt.try_dispatch(
copt_c_sources,
output_dir=output_dir,
src_dir=copt_build_src,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args,
**kws)
if c_sources:
log.info("compiling C sources")
c_objects += self.compiler.compile(c_sources,
output_dir=output_dir,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args +
copt_baseline_flags,
**kws)
if cxx_sources:
log.info("compiling C++ sources")
c_objects += cxx_compiler.compile(cxx_sources,
output_dir=output_dir,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args +
copt_baseline_flags,
**kws)
extra_postargs = []
f_objects = []
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
module_dirs = ext.module_dirs[:]
module_build_dir = os.path.join(
self.build_temp,
os.path.dirname(self.get_ext_filename(fullname)))
self.mkpath(module_build_dir)
if fcompiler.module_dir_switch is None:
existing_modules = glob("*.mod")
extra_postargs += fcompiler.module_options(module_dirs,
module_build_dir)
f_objects += fcompiler.compile(
fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends,
)
if fcompiler.module_dir_switch is None:
for f in glob("*.mod"):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f) == os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn("failed to move %r to %r" %
(f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(
f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends,
)
if f_objects and not fcompiler.can_ccompiler_link(self.compiler):
unlinkable_fobjects = f_objects
objects = c_objects
else:
unlinkable_fobjects = []
objects = c_objects + f_objects
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
libraries = self.get_libraries(ext)[:]
library_dirs = ext.library_dirs[:]
linker = self.compiler.link_shared_object
# Always use system linker when using MSVC compiler.
if self.compiler.compiler_type in ("msvc", "intelw", "intelemw"):
# expand libraries with fcompiler libraries as we are
# not using fcompiler linker
self._libs_with_msvc_and_fortran(fcompiler, libraries,
library_dirs)
elif ext.language in ["f77", "f90"] and fcompiler is not None:
linker = fcompiler.link_shared_object
if ext.language == "c++" and cxx_compiler is not None:
linker = cxx_compiler.link_shared_object
if fcompiler is not None:
objects, libraries = self._process_unlinkable_fobjects(
objects, libraries, fcompiler, library_dirs,
unlinkable_fobjects)
linker(
objects,
ext_filename,
libraries=libraries,
library_dirs=library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=ext.language,
)
def _exec_command(command, use_shell=None, use_tee=None, **env):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name == 'posix'
if use_tee is None:
use_tee = os.name == 'posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
argv = [sh, '-c', ' '.join(list(command))]
else:
argv = [sh, '-c', command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
if hasattr(os, 'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt', 'dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'], '/C'] + argv
using_command = 1
_so_has_fileno = _supports_fileno(sys.stdout)
_se_has_fileno = _supports_fileno(sys.stderr)
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
if _so_has_fileno:
so_fileno = sys.stdout.fileno()
so_dup = os.dup(so_fileno)
if _se_has_fileno:
se_fileno = sys.stderr.fileno()
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile, 'w')
if using_command:
errfile = temp_file_name()
ferr = open(errfile, 'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__, os.P_WAIT, argv[0], argv))
if sys.version_info[0] >= 3 and os.name == 'nt':
# Pre-encode os.environ, discarding un-encodable entries,
# to avoid it failing during encoding as part of spawn. Failure
# is possible if the environment contains entries that are not
# encoded using the system codepage as windows expects.
#
# This is not necessary on unix, where os.environ is encoded
# using the surrogateescape error handler and decoded using
# it as part of spawn.
encoded_environ = {}
for k, v in os.environ.items():
try:
encoded_environ[k.encode(
sys.getfilesystemencoding())] = v.encode(
sys.getfilesystemencoding())
except UnicodeEncodeError:
log.debug("ignoring un-encodable env entry %s", k)
else:
encoded_environ = os.environ
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(fout.fileno(), so_fileno)
if _se_has_fileno:
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(), se_fileno)
else:
os.dup2(fout.fileno(), se_fileno)
try:
status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ)
except Exception:
errmess = str(get_exception())
status = 999
sys.stderr.write('%s: %s' % (errmess, argv[0]))
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(so_dup, so_fileno)
os.close(so_dup)
if _se_has_fileno:
os.dup2(se_dup, se_fileno)
os.close(se_dup)
fout.close()
fout = open_latin1(outfile, 'r')
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile, 'r')
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
#status = 998
if text:
text = text + '\n'
#text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print(errmess)
if text[-1:] == '\n':
text = text[:-1]
if status is None:
status = 0
if use_tee:
print(text)
return status, text
def setup(**attr):
if len(sys.argv) <= 1 and not attr.get("script_args", []):
from interactive import interactive_sys_argv
import atexit
atexit.register(_exit_interactive_session)
sys.argv[:] = interactive_sys_argv(sys.argv)
if len(sys.argv) > 1:
return setup(**attr)
cmdclass = numpy_cmdclass.copy()
new_attr = attr.copy()
if "cmdclass" in new_attr:
cmdclass.update(new_attr["cmdclass"])
new_attr["cmdclass"] = cmdclass
if "configuration" in new_attr:
# To avoid calling configuration if there are any errors
# or help request in command in the line.
configuration = new_attr.pop("configuration")
old_dist = distutils.core._setup_distribution
old_stop = distutils.core._setup_stop_after
distutils.core._setup_distribution = None
distutils.core._setup_stop_after = "commandline"
try:
dist = setup(**new_attr)
finally:
distutils.core._setup_distribution = old_dist
distutils.core._setup_stop_after = old_stop
if dist.help or not _command_line_ok():
# probably displayed help, skip running any commands
return dist
# create setup dictionary and append to new_attr
config = configuration()
if hasattr(config, "todict"):
config = config.todict()
_dict_append(new_attr, **config)
# Move extension source libraries to libraries
libraries = []
for ext in new_attr.get("ext_modules", []):
new_libraries = []
for item in ext.libraries:
if is_sequence(item):
lib_name, build_info = item
_check_append_ext_library(libraries, lib_name, build_info)
new_libraries.append(lib_name)
elif is_string(item):
new_libraries.append(item)
else:
raise TypeError("invalid description of extension module " "library %r" % (item,))
ext.libraries = new_libraries
if libraries:
if "libraries" not in new_attr:
new_attr["libraries"] = []
for item in libraries:
_check_append_library(new_attr["libraries"], item)
# sources in ext_modules or libraries may contain header files
if ("ext_modules" in new_attr or "libraries" in new_attr) and "headers" not in new_attr:
new_attr["headers"] = []
# Use our custom NumpyDistribution class instead of distutils' one
new_attr["distclass"] = NumpyDistribution
return old_setup(**new_attr)
def f2py_sources(self, sources, extension):
new_sources = []
f2py_sources = []
f_sources = []
f2py_targets = {}
target_dirs = []
ext_name = extension.name.split('.')[-1]
skip_f2py = 0
for source in sources:
(base, ext) = os.path.splitext(source)
if ext == '.pyf': # F2PY interface file
if self.inplace:
target_dir = os.path.dirname(base)
else:
target_dir = appendpath(self.build_src, os.path.dirname(base))
if os.path.isfile(source):
name = get_f2py_modulename(source)
if name != ext_name:
raise DistutilsSetupError('mismatch of extension names: %s '
'provides %r but expected %r' % (
source, name, ext_name))
target_file = os.path.join(target_dir,name+'module.c')
else:
log.debug(' source %s does not exist: skipping f2py\'ing.' \
% (source))
name = ext_name
skip_f2py = 1
target_file = os.path.join(target_dir,name+'module.c')
if not os.path.isfile(target_file):
log.warn(' target %s does not exist:\n '\
'Assuming %smodule.c was generated with '\
'"build_src --inplace" command.' \
% (target_file, name))
target_dir = os.path.dirname(base)
target_file = os.path.join(target_dir,name+'module.c')
if not os.path.isfile(target_file):
raise DistutilsSetupError("%r missing" % (target_file,))
log.info(' Yes! Using %r as up-to-date target.' \
% (target_file))
target_dirs.append(target_dir)
f2py_sources.append(source)
f2py_targets[source] = target_file
new_sources.append(target_file)
elif fortran_ext_match(ext):
f_sources.append(source)
else:
new_sources.append(source)
if not (f2py_sources or f_sources):
return new_sources
map(self.mkpath, target_dirs)
f2py_options = extension.f2py_options + self.f2py_opts
if self.distribution.libraries:
for name,build_info in self.distribution.libraries:
if name in extension.libraries:
f2py_options.extend(build_info.get('f2py_options',[]))
log.info("f2py options: %s" % (f2py_options))
if f2py_sources:
if len(f2py_sources) != 1:
raise DistutilsSetupError(
'only one .pyf file is allowed per extension module but got'\
' more: %r' % (f2py_sources,))
source = f2py_sources[0]
target_file = f2py_targets[source]
target_dir = os.path.dirname(target_file) or '.'
depends = [source] + extension.depends
if (self.force or newer_group(depends, target_file,'newer')) \
and not skip_f2py:
log.info("f2py: %s" % (source))
import numpy.f2py as f2py2e
f2py2e.run_main(f2py_options + ['--build-dir',target_dir,source])
else:
log.debug(" skipping '%s' f2py interface (up-to-date)" % (source))
else:
#XXX TODO: --inplace support for sdist command
if is_sequence(extension):
name = extension[0]
else: name = extension.name
target_dir = os.path.join(*([self.build_src]\
+name.split('.')[:-1]))
target_file = os.path.join(target_dir,ext_name + 'module.c')
new_sources.append(target_file)
depends = f_sources + extension.depends
if (self.force or newer_group(depends, target_file, 'newer')) \
and not skip_f2py:
import numpy.f2py as f2py2e
log.info("f2py:> %s" % (target_file))
self.mkpath(target_dir)
f2py2e.run_main(f2py_options + ['--lower',
'--build-dir',target_dir]+\
['-m',ext_name]+f_sources)
else:
log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\
% (target_file))
if not os.path.isfile(target_file):
raise DistutilsError("f2py target file %r not generated" % (target_file,))
target_c = os.path.join(self.build_src,'fortranobject.c')
target_h = os.path.join(self.build_src,'fortranobject.h')
log.info(" adding '%s' to sources." % (target_c))
new_sources.append(target_c)
if self.build_src not in extension.include_dirs:
log.info(" adding '%s' to include_dirs." \
% (self.build_src))
extension.include_dirs.append(self.build_src)
if not skip_f2py:
import numpy.f2py as f2py2e
d = os.path.dirname(f2py2e.__file__)
source_c = os.path.join(d,'src','fortranobject.c')
source_h = os.path.join(d,'src','fortranobject.h')
if newer(source_c,target_c) or newer(source_h,target_h):
self.mkpath(os.path.dirname(target_c))
self.copy_file(source_c,target_c)
self.copy_file(source_h,target_h)
else:
if not os.path.isfile(target_c):
raise DistutilsSetupError("f2py target_c file %r not found" % (target_c,))
if not os.path.isfile(target_h):
raise DistutilsSetupError("f2py target_h file %r not found" % (target_h,))
for name_ext in ['-f2pywrappers.f','-f2pywrappers2.f90']:
filename = os.path.join(target_dir,ext_name + name_ext)
if os.path.isfile(filename):
log.info(" adding '%s' to sources." % (filename))
f_sources.append(filename)
return new_sources + f_sources
def _exec_command(command, use_shell=None, use_tee=None, **env):
log.debug("_exec_command(...)")
if use_shell is None:
use_shell = os.name == "posix"
if use_tee is None:
use_tee = os.name == "posix"
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get("SHELL", "/bin/sh")
if is_sequence(command):
argv = [sh, "-c", " ".join(list(command))]
else:
argv = [sh, "-c", command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
# `spawn*p` family with path (vp, vpe, ...) are not available on windows.
# Also prefer spawn{v,vp} in favor of spawn{ve,vpe} if no env
# modification is actually requested as the *e* functions are not thread
# safe on windows (https://bugs.python.org/issue6476)
if hasattr(os, "spawnvpe"):
spawn_command = os.spawnvpe if env else os.spawnvp
else:
spawn_command = os.spawnve if env else os.spawnv
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn("Executable %s does not exist" % (argv[0]))
if os.name in ["nt", "dos"]:
# argv[0] might be internal command
argv = [os.environ["COMSPEC"], "/C"] + argv
using_command = 1
_so_has_fileno = _supports_fileno(sys.stdout)
_se_has_fileno = _supports_fileno(sys.stderr)
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
if _so_has_fileno:
so_fileno = sys.stdout.fileno()
so_dup = os.dup(so_fileno)
if _se_has_fileno:
se_fileno = sys.stderr.fileno()
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile, "w")
if using_command:
errfile = temp_file_name()
ferr = open(errfile, "w")
log.debug("Running %s(%s,%r,%r,os.environ)" % (spawn_command.__name__, os.P_WAIT, argv[0], argv))
if env and sys.version_info[0] >= 3 and os.name == "nt":
# Pre-encode os.environ, discarding un-encodable entries,
# to avoid it failing during encoding as part of spawn. Failure
# is possible if the environment contains entries that are not
# encoded using the system codepage as windows expects.
#
# This is not necessary on unix, where os.environ is encoded
# using the surrogateescape error handler and decoded using
# it as part of spawn.
encoded_environ = {}
for k, v in os.environ.items():
try:
encoded_environ[k.encode(sys.getfilesystemencoding())] = v.encode(sys.getfilesystemencoding())
except UnicodeEncodeError:
log.debug("ignoring un-encodable env entry %s", k)
else:
encoded_environ = os.environ
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(fout.fileno(), so_fileno)
if _se_has_fileno:
if using_command:
# XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(), se_fileno)
else:
os.dup2(fout.fileno(), se_fileno)
try:
# Use spawnv in favor of spawnve, unless necessary
if env:
status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ)
else:
status = spawn_command(os.P_WAIT, argv0, argv)
except Exception:
errmess = str(get_exception())
status = 999
sys.stderr.write("%s: %s" % (errmess, argv[0]))
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(so_dup, so_fileno)
os.close(so_dup)
if _se_has_fileno:
os.dup2(se_dup, se_fileno)
os.close(se_dup)
fout.close()
fout = open_latin1(outfile, "r")
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile, "r")
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
# status = 998
if text:
text = text + "\n"
# text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print(errmess)
if text[-1:] == "\n":
text = text[:-1]
if status is None:
status = 0
if use_tee:
print(text)
return status, text
def setup(**attr):
cmdclass = numpy_cmdclass.copy()
new_attr = attr.copy()
if 'cmdclass' in new_attr:
cmdclass.update(new_attr['cmdclass'])
new_attr['cmdclass'] = cmdclass
if 'configuration' in new_attr:
# To avoid calling configuration if there are any errors
# or help request in command in the line.
configuration = new_attr.pop('configuration')
old_dist = distutils.core._setup_distribution
old_stop = distutils.core._setup_stop_after
distutils.core._setup_distribution = None
distutils.core._setup_stop_after = "commandline"
try:
dist = setup(**new_attr)
finally:
distutils.core._setup_distribution = old_dist
distutils.core._setup_stop_after = old_stop
if dist.help or not _command_line_ok():
# probably displayed help, skip running any commands
return dist
# create setup dictionary and append to new_attr
config = configuration()
if hasattr(config, 'todict'):
config = config.todict()
_dict_append(new_attr, **config)
# Move extension source libraries to libraries
libraries = []
for ext in new_attr.get('ext_modules', []):
new_libraries = []
for item in ext.libraries:
if is_sequence(item):
lib_name, build_info = item
_check_append_ext_library(libraries, lib_name, build_info)
new_libraries.append(lib_name)
elif is_string(item):
new_libraries.append(item)
else:
raise TypeError("invalid description of extension module "
"library %r" % (item, ))
ext.libraries = new_libraries
if libraries:
if 'libraries' not in new_attr:
new_attr['libraries'] = []
for item in libraries:
_check_append_library(new_attr['libraries'], item)
# sources in ext_modules or libraries may contain header files
if ('ext_modules' in new_attr or 'libraries' in new_attr) \
and 'headers' not in new_attr:
new_attr['headers'] = []
# Use our custom NumpyDistribution class instead of distutils' one
new_attr['distclass'] = NumpyDistribution
return old_setup(**new_attr)
def is_sequence_of_strings(seq):
return is_sequence(seq) and all_strings(seq)
def _fix_args(args, flag=1):
if is_string(args):
return args.replace('%', '%%')
if flag and is_sequence(args):
return tuple([_fix_args(a, flag=0) for a in args])
return args
def is_sequence_of_strings(seq):
return is_sequence(seq) and all_strings(seq)
def build_extension(self, ext):
sources = ext.sources
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(
("in 'ext_modules' option (extension '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % ext.name)
sources = list(sources)
if not sources:
return
fullname = self.get_ext_fullname(ext.name)
if self.inplace:
modpath = fullname.split('.')
package = '.'.join(modpath[0:-1])
base = modpath[-1]
build_py = self.get_finalized_command('build_py')
package_dir = build_py.get_package_dir(package)
ext_filename = os.path.join(package_dir,
self.get_ext_filename(base))
else:
ext_filename = os.path.join(self.build_lib,
self.get_ext_filename(fullname))
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_filename, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef, ))
c_sources, cxx_sources, f_sources, fmodule_sources = \
filter_sources(ext.sources)
if self.compiler.compiler_type == 'msvc':
if cxx_sources:
# Needed to compile kiva.agg._agg extension.
extra_args.append('/Zm1000')
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
# Set Fortran/C++ compilers for compilation and linking.
if ext.language == 'f90':
fcompiler = self._f90_compiler
elif ext.language == 'f77':
fcompiler = self._f77_compiler
else: # in case ext.language is c++, for instance
fcompiler = self._f90_compiler or self._f77_compiler
if fcompiler is not None:
fcompiler.extra_f77_compile_args = (
ext.extra_f77_compile_args or []) if hasattr(
ext, 'extra_f77_compile_args') else []
fcompiler.extra_f90_compile_args = (
ext.extra_f90_compile_args or []) if hasattr(
ext, 'extra_f90_compile_args') else []
cxx_compiler = self._cxx_compiler
# check for the availability of required compilers
if cxx_sources and cxx_compiler is None:
raise DistutilsError("extension %r has C++ sources" \
"but no C++ compiler found" % (ext.name))
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("extension %r has Fortran sources " \
"but no Fortran compiler found" % (ext.name))
if ext.language in ['f77', 'f90'] and fcompiler is None:
self.warn("extension %r has Fortran libraries " \
"but no Fortran linker found, using default linker" % (ext.name))
if ext.language == 'c++' and cxx_compiler is None:
self.warn("extension %r has C++ libraries " \
"but no C++ linker found, using default linker" % (ext.name))
kws = {'depends': ext.depends}
output_dir = self.build_temp
include_dirs = ext.include_dirs + get_numpy_include_dirs()
c_objects = []
if c_sources:
log.info("compiling C sources")
c_objects = self.compiler.compile(c_sources,
output_dir=output_dir,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args,
**kws)
if cxx_sources:
log.info("compiling C++ sources")
c_objects += cxx_compiler.compile(cxx_sources,
output_dir=output_dir,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args,
**kws)
extra_postargs = []
f_objects = []
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
module_dirs = ext.module_dirs[:]
module_build_dir = os.path.join(
self.build_temp,
os.path.dirname(self.get_ext_filename(fullname)))
self.mkpath(module_build_dir)
if fcompiler.module_dir_switch is None:
existing_modules = glob('*.mod')
extra_postargs += fcompiler.module_options(module_dirs,
module_build_dir)
f_objects += fcompiler.compile(fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends)
if fcompiler.module_dir_switch is None:
for f in glob('*.mod'):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f) == os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn('failed to move %r to %r' %
(f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends)
objects = c_objects + f_objects
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
libraries = self.get_libraries(ext)[:]
library_dirs = ext.library_dirs[:]
linker = self.compiler.link_shared_object
# Always use system linker when using MSVC compiler.
if self.compiler.compiler_type == 'msvc':
# expand libraries with fcompiler libraries as we are
# not using fcompiler linker
self._libs_with_msvc_and_fortran(fcompiler, libraries,
library_dirs)
elif ext.language in ['f77', 'f90'] and fcompiler is not None:
linker = fcompiler.link_shared_object
if ext.language == 'c++' and cxx_compiler is not None:
linker = cxx_compiler.link_shared_object
linker(objects,
ext_filename,
libraries=libraries,
library_dirs=library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=ext.language)
def build_extension(self, ext):
sources = ext.sources
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(
("in 'ext_modules' option (extension '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % ext.name)
sources = list(sources)
if not sources:
return
fullname = self.get_ext_fullname(ext.name)
if self.inplace:
modpath = fullname.split('.')
package = '.'.join(modpath[0:-1])
base = modpath[-1]
build_py = self.get_finalized_command('build_py')
package_dir = build_py.get_package_dir(package)
ext_filename = os.path.join(package_dir,
self.get_ext_filename(base))
else:
ext_filename = os.path.join(self.build_lib,
self.get_ext_filename(fullname))
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_filename, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef,))
c_sources, cxx_sources, f_sources, fmodule_sources = \
filter_sources(ext.sources)
if self.compiler.compiler_type == 'msvc':
if cxx_sources:
# Needed to compile kiva.agg._agg extension.
extra_args.append('/Zm1000')
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
# Set Fortran/C++ compilers for compilation and linking.
if ext.language == 'f90':
fcompiler = self._f90_compiler
elif ext.language == 'f77':
fcompiler = self._f77_compiler
else: # in case ext.language is c++, for instance
fcompiler = self._f90_compiler or self._f77_compiler
if fcompiler is not None:
fcompiler.extra_f77_compile_args = (ext.extra_f77_compile_args or []) if hasattr(
ext, 'extra_f77_compile_args') else []
fcompiler.extra_f90_compile_args = (ext.extra_f90_compile_args or []) if hasattr(
ext, 'extra_f90_compile_args') else []
cxx_compiler = self._cxx_compiler
# check for the availability of required compilers
if cxx_sources and cxx_compiler is None:
raise DistutilsError("extension %r has C++ sources"
"but no C++ compiler found" % (ext.name))
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("extension %r has Fortran sources "
"but no Fortran compiler found" % (ext.name))
if ext.language in ['f77', 'f90'] and fcompiler is None:
self.warn("extension %r has Fortran libraries "
"but no Fortran linker found, using default linker" % (ext.name))
if ext.language == 'c++' and cxx_compiler is None:
self.warn("extension %r has C++ libraries "
"but no C++ linker found, using default linker" % (ext.name))
kws = {'depends': ext.depends}
output_dir = self.build_temp
include_dirs = ext.include_dirs + get_numpy_include_dirs()
c_objects = []
if c_sources:
log.info("compiling C sources")
c_objects = self.compiler.compile(c_sources,
output_dir=output_dir,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args,
**kws)
if cxx_sources:
log.info("compiling C++ sources")
c_objects += cxx_compiler.compile(cxx_sources,
output_dir=output_dir,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_args,
**kws)
extra_postargs = []
f_objects = []
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
module_dirs = ext.module_dirs[:]
module_build_dir = os.path.join(
self.build_temp, os.path.dirname(
self.get_ext_filename(fullname)))
self.mkpath(module_build_dir)
if fcompiler.module_dir_switch is None:
existing_modules = glob('*.mod')
extra_postargs += fcompiler.module_options(
module_dirs, module_build_dir)
f_objects += fcompiler.compile(fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends)
if fcompiler.module_dir_switch is None:
for f in glob('*.mod'):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f) == os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn('failed to move %r to %r' %
(f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
depends=ext.depends)
if f_objects and not fcompiler.can_ccompiler_link(self.compiler):
unlinkable_fobjects = f_objects
objects = c_objects
else:
unlinkable_fobjects = []
objects = c_objects + f_objects
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
libraries = self.get_libraries(ext)[:]
library_dirs = ext.library_dirs[:]
linker = self.compiler.link_shared_object
# Always use system linker when using MSVC compiler.
if self.compiler.compiler_type in ('msvc', 'intelw', 'intelemw'):
# expand libraries with fcompiler libraries as we are
# not using fcompiler linker
self._libs_with_msvc_and_fortran(
fcompiler, libraries, library_dirs)
elif ext.language in ['f77', 'f90'] and fcompiler is not None:
linker = fcompiler.link_shared_object
if ext.language == 'c++' and cxx_compiler is not None:
linker = cxx_compiler.link_shared_object
if fcompiler is not None:
objects, libraries = self._process_unlinkable_fobjects(
objects, libraries,
fcompiler, library_dirs,
unlinkable_fobjects)
linker(objects, ext_filename,
libraries=libraries,
library_dirs=library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=ext.language)
def _exec_command(command, use_shell=None, use_tee=None, **env):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name == 'posix'
if use_tee is None:
use_tee = os.name == 'posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
argv = [sh, '-c', ' '.join(list(command))]
else:
argv = [sh, '-c', command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
if hasattr(os, 'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt', 'dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'], '/C'] + argv
using_command = 1
_so_has_fileno = _supports_fileno(sys.stdout)
_se_has_fileno = _supports_fileno(sys.stderr)
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
if _so_has_fileno:
so_fileno = sys.stdout.fileno()
so_dup = os.dup(so_fileno)
if _se_has_fileno:
se_fileno = sys.stderr.fileno()
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile, 'w')
if using_command:
errfile = temp_file_name()
ferr = open(errfile, 'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__,os.P_WAIT,argv[0],argv))
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(fout.fileno(), so_fileno)
if _se_has_fileno:
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(), se_fileno)
else:
os.dup2(fout.fileno(), se_fileno)
try:
status = spawn_command(os.P_WAIT, argv0, argv, os.environ)
except OSError:
errmess = str(get_exception())
status = 999
sys.stderr.write('%s: %s' % (errmess, argv[0]))
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(so_dup, so_fileno)
if _se_has_fileno:
os.dup2(se_dup, se_fileno)
fout.close()
fout = open_latin1(outfile, 'r')
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile, 'r')
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
#status = 998
if text:
text = text + '\n'
#text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print(errmess)
if text[-1:] == '\n':
text = text[:-1]
if status is None:
status = 0
if use_tee:
print(text)
return status, text
def _load_file_and_cmd(self) -> None:
"""
Load settings from local file and arguments of the command line.
"""
def str_to_bool(arg_value: str) -> bool:
"""
Used to handle boolean settings.
If not the 'bool' type convert all not empty string as true.
:param arg_value: The boolean value as a string.
:return: The value parsed as a string.
"""
if isinstance(arg_value, bool):
return arg_value
if arg_value.lower() in {'false', 'f', '0', 'no', 'n'}:
return False
elif arg_value.lower() in {'true', 't', '1', 'yes', 'y'}:
return True
raise argparse.ArgumentTypeError(
f'{arg_value} is not a valid boolean value')
def type_mapping(arg_value):
if type(arg_value) == bool:
return str_to_bool
if is_sequence(arg_value):
if len(arg_value) == 0:
return str
else:
return type_mapping(arg_value[0])
# Default same as current value
return type(arg_value)
p = configargparse.get_argument_parser(
default_config_files=['./settings.yaml'])
# Spacial argument
p.add_argument('-s',
'--settings',
required=False,
is_config_file=True,
help='path to custom configuration file')
# Create argument for each attribute of this class
for name, value in asdict(self).items():
p.add_argument(f'--{name.replace("_", "-")}',
f'--{name}',
dest=name,
required=False,
action='append' if is_sequence(value) else 'store',
type=type_mapping(value))
# Load arguments form file, environment and command line to override the defaults
for name, value in vars(p.parse_args()).items():
if name == 'settings':
continue
if value is not None:
# Directly set the value to bypass the "__setattr__" function
self.__dict__[name] = value
self.validate()
def CCompiler_spawn(self, cmd, display=None):
"""
Execute a command in a sub-process.
Parameters
----------
cmd : str
The command to execute.
display : str or sequence of str, optional
The text to add to the log file kept by `numpy.distutils`.
If not given, `display` is equal to `cmd`.
Returns
-------
None
Raises
------
DistutilsExecError
If the command failed, i.e. the exit status was not 0.
"""
if display is None:
display = cmd
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
try:
if self.verbose:
subprocess.check_output(cmd)
else:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
o = exc.output
s = exc.returncode
except OSError:
# OSError doesn't have the same hooks for the exception
# output, but exec_command() historically would use an
# empty string for EnvironmentError (base class for
# OSError)
o = b''
# status previously used by exec_command() for parent
# of OSError
s = 127
else:
# use a convenience return here so that any kind of
# caught exception will execute the default code after the
# try / except block, which handles various exceptions
return None
if is_sequence(cmd):
cmd = ' '.join(list(cmd))
if self.verbose:
forward_bytes_to_stdout(o)
if re.search(b'Too many open files', o):
msg = '\nTry rerunning setup command until build succeeds.'
else:
msg = ''
raise DistutilsExecError('Command "%s" failed with exit status %d%s' %
(cmd, s, msg))
def setup(**attr):
cmdclass = numpy_cmdclass.copy()
new_attr = attr.copy()
if 'cmdclass' in new_attr:
cmdclass.update(new_attr['cmdclass'])
new_attr['cmdclass'] = cmdclass
if 'configuration' in new_attr:
# To avoid calling configuration if there are any errors
# or help request in command in the line.
configuration = new_attr.pop('configuration')
old_dist = distutils.core._setup_distribution
old_stop = distutils.core._setup_stop_after
distutils.core._setup_distribution = None
distutils.core._setup_stop_after = "commandline"
try:
dist = setup(**new_attr)
finally:
distutils.core._setup_distribution = old_dist
distutils.core._setup_stop_after = old_stop
if dist.help or not _command_line_ok():
# probably displayed help, skip running any commands
return dist
# create setup dictionary and append to new_attr
config = configuration()
if hasattr(config,'todict'):
config = config.todict()
_dict_append(new_attr, **config)
# Move extension source libraries to libraries
libraries = []
for ext in new_attr.get('ext_modules',[]):
new_libraries = []
for item in ext.libraries:
if is_sequence(item):
lib_name, build_info = item
_check_append_ext_library(libraries, lib_name, build_info)
new_libraries.append(lib_name)
elif is_string(item):
new_libraries.append(item)
else:
raise TypeError("invalid description of extension module "
"library %r" % (item,))
ext.libraries = new_libraries
if libraries:
if 'libraries' not in new_attr:
new_attr['libraries'] = []
for item in libraries:
_check_append_library(new_attr['libraries'], item)
# sources in ext_modules or libraries may contain header files
if ('ext_modules' in new_attr or 'libraries' in new_attr) \
and 'headers' not in new_attr:
new_attr['headers'] = []
# Use our custom NumpyDistribution class instead of distutils' one
new_attr['distclass'] = NumpyDistribution
return old_setup(**new_attr)
def _exec_command( command, use_shell=None, use_tee = None, **env ):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
argv = [sh, '-c', ' '.join(list(command))]
else:
argv = [sh, '-c', command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
if hasattr(os, 'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt', 'dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'], '/C'] + argv
using_command = 1
_so_has_fileno = _supports_fileno(sys.stdout)
_se_has_fileno = _supports_fileno(sys.stderr)
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
if _so_has_fileno:
so_fileno = sys.stdout.fileno()
so_dup = os.dup(so_fileno)
if _se_has_fileno:
se_fileno = sys.stderr.fileno()
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile, 'w')
if using_command:
errfile = temp_file_name()
ferr = open(errfile, 'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__, os.P_WAIT, argv[0], argv))
if sys.version_info[0] >= 3 and os.name == 'nt':
# Pre-encode os.environ, discarding un-encodable entries,
# to avoid it failing during encoding as part of spawn. Failure
# is possible if the environment contains entries that are not
# encoded using the system codepage as windows expects.
#
# This is not necessary on unix, where os.environ is encoded
# using the surrogateescape error handler and decoded using
# it as part of spawn.
encoded_environ = {}
for k, v in os.environ.items():
try:
encoded_environ[k.encode(sys.getfilesystemencoding())] = v.encode(
sys.getfilesystemencoding())
except UnicodeEncodeError:
log.debug("ignoring un-encodable env entry %s", k)
else:
encoded_environ = os.environ
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(fout.fileno(), so_fileno)
if _se_has_fileno:
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(), se_fileno)
else:
os.dup2(fout.fileno(), se_fileno)
try:
status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ)
except Exception:
errmess = str(get_exception())
status = 999
sys.stderr.write('%s: %s'%(errmess, argv[0]))
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(so_dup, so_fileno)
os.close(so_dup)
if _se_has_fileno:
os.dup2(se_dup, se_fileno)
os.close(se_dup)
fout.close()
fout = open_latin1(outfile, 'r')
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile, 'r')
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
#status = 998
if text:
text = text + '\n'
#text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print (errmess)
if text[-1:]=='\n':
text = text[:-1]
if status is None:
status = 0
if use_tee:
print (text)
return status, text
def build_a_library(self, build_info, lib_name, libraries):
# default compilers
compiler = self.compiler
fcompiler = self._f_compiler
sources = build_info.get("sources")
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(
("in 'libraries' option (library '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % lib_name)
sources = list(sources)
c_sources, cxx_sources, f_sources, fmodule_sources = filter_sources(
sources)
requiref90 = not not fmodule_sources or build_info.get(
"language", "c") == "f90"
# save source type information so that build_ext can use it.
source_languages = []
if c_sources:
source_languages.append("c")
if cxx_sources:
source_languages.append("c++")
if requiref90:
source_languages.append("f90")
elif f_sources:
source_languages.append("f77")
build_info["source_languages"] = source_languages
lib_file = compiler.library_filename(lib_name,
output_dir=self.build_clib)
depends = sources + build_info.get("depends", [])
if not (self.force or newer_group(depends, lib_file, "newer")):
log.debug("skipping '%s' library (up-to-date)", lib_name)
return
else:
log.info("building '%s' library", lib_name)
config_fc = build_info.get("config_fc", {})
if fcompiler is not None and config_fc:
log.info("using additional config_fc from setup script "
"for fortran compiler: %s" % (config_fc, ))
from numpy.distutils.fcompiler import new_fcompiler
fcompiler = new_fcompiler(
compiler=fcompiler.compiler_type,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90=requiref90,
c_compiler=self.compiler,
)
if fcompiler is not None:
dist = self.distribution
base_config_fc = dist.get_option_dict("config_fc").copy()
base_config_fc.update(config_fc)
fcompiler.customize(base_config_fc)
# check availability of Fortran compilers
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("library %s has Fortran sources"
" but no Fortran compiler found" % (lib_name))
if fcompiler is not None:
fcompiler.extra_f77_compile_args = (
build_info.get("extra_f77_compile_args") or [])
fcompiler.extra_f90_compile_args = (
build_info.get("extra_f90_compile_args") or [])
macros = build_info.get("macros")
if macros is None:
macros = []
include_dirs = build_info.get("include_dirs")
if include_dirs is None:
include_dirs = []
extra_postargs = build_info.get("extra_compiler_args") or []
include_dirs.extend(get_numpy_include_dirs())
# where compiled F90 module files are:
module_dirs = build_info.get("module_dirs") or []
module_build_dir = os.path.dirname(lib_file)
if requiref90:
self.mkpath(module_build_dir)
if compiler.compiler_type == "msvc":
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
# filtering C dispatch-table sources when optimization is not disabled,
# otherwise treated as normal sources.
copt_c_sources = []
copt_baseline_flags = []
copt_macros = []
if not self.disable_optimization:
bsrc_dir = self.get_finalized_command("build_src").build_src
dispatch_hpath = os.path.join("numpy", "distutils", "include")
dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath)
include_dirs.append(dispatch_hpath)
copt_build_src = None if self.inplace else bsrc_dir
copt_c_sources = [
c_sources.pop(c_sources.index(src)) for src in c_sources[:]
if src.endswith(".dispatch.c")
]
copt_baseline_flags = self.compiler_opt.cpu_baseline_flags()
else:
copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1))
objects = []
if copt_c_sources:
log.info("compiling C dispatch-able sources")
objects += self.compiler_opt.try_dispatch(
copt_c_sources,
output_dir=self.build_temp,
src_dir=copt_build_src,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
)
if c_sources:
log.info("compiling C sources")
objects += compiler.compile(
c_sources,
output_dir=self.build_temp,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs + copt_baseline_flags,
)
if cxx_sources:
log.info("compiling C++ sources")
cxx_compiler = compiler.cxx_compiler()
cxx_objects = cxx_compiler.compile(
cxx_sources,
output_dir=self.build_temp,
macros=macros + copt_macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs + copt_baseline_flags,
)
objects.extend(cxx_objects)
if f_sources or fmodule_sources:
extra_postargs = []
f_objects = []
if requiref90:
if fcompiler.module_dir_switch is None:
existing_modules = glob("*.mod")
extra_postargs += fcompiler.module_options(
module_dirs, module_build_dir)
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
f_objects += fcompiler.compile(
fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
)
if requiref90 and self._f_compiler.module_dir_switch is None:
# move new compiled F90 module files to module_build_dir
for f in glob("*.mod"):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f) == os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn("failed to move %r to %r" %
(f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(
f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs,
)
else:
f_objects = []
if f_objects and not fcompiler.can_ccompiler_link(compiler):
# Default linker cannot link Fortran object files, and results
# need to be wrapped later. Instead of creating a real static
# library, just keep track of the object files.
listfn = os.path.join(self.build_clib, lib_name + ".fobjects")
with open(listfn, "w") as f:
f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
listfn = os.path.join(self.build_clib, lib_name + ".cobjects")
with open(listfn, "w") as f:
f.write("\n".join(os.path.abspath(obj) for obj in objects))
# create empty "library" file for dependency tracking
lib_fname = os.path.join(self.build_clib,
lib_name + compiler.static_lib_extension)
with open(lib_fname, "wb") as f:
pass
else:
# assume that default linker is suitable for
# linking Fortran object files
objects.extend(f_objects)
compiler.create_static_lib(objects,
lib_name,
output_dir=self.build_clib,
debug=self.debug)
# fix library dependencies
clib_libraries = build_info.get("libraries", [])
for lname, binfo in libraries:
if lname in clib_libraries:
clib_libraries.extend(binfo.get("libraries", []))
if clib_libraries:
build_info["libraries"] = clib_libraries
def _fix_args(args,flag=1):
if is_string(args):
return args.replace('%','%%')
if flag and is_sequence(args):
return tuple([_fix_args(a,flag=0) for a in args])
return args
def _exec_command( command, use_shell=None, use_tee = None, **env ):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL', '/bin/sh')
if is_sequence(command):
argv = [sh, '-c', ' '.join(list(command))]
else:
argv = [sh, '-c', command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
if hasattr(os, 'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt', 'dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'], '/C'] + argv
using_command = 1
_so_has_fileno = _supports_fileno(sys.stdout)
_se_has_fileno = _supports_fileno(sys.stderr)
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
if _so_has_fileno:
so_fileno = sys.stdout.fileno()
so_dup = os.dup(so_fileno)
if _se_has_fileno:
se_fileno = sys.stderr.fileno()
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile, 'w')
if using_command:
errfile = temp_file_name()
ferr = open(errfile, 'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__, os.P_WAIT, argv[0], argv))
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(fout.fileno(), so_fileno)
if _se_has_fileno:
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(), se_fileno)
else:
os.dup2(fout.fileno(), se_fileno)
try:
status = spawn_command(os.P_WAIT, argv0, argv, os.environ)
except OSError:
errmess = str(get_exception())
status = 999
sys.stderr.write('%s: %s'%(errmess, argv[0]))
so_flush()
se_flush()
if _so_has_fileno:
os.dup2(so_dup, so_fileno)
if _se_has_fileno:
os.dup2(se_dup, se_fileno)
fout.close()
fout = open_latin1(outfile, 'r')
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile, 'r')
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
#status = 998
if text:
text = text + '\n'
#text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print (errmess)
if text[-1:]=='\n':
text = text[:-1]
if status is None:
status = 0
if use_tee:
print (text)
return status, text
else:
if item==libitem[0]:
warnings.warn("[1] libraries list contains %r with"
" no build_info" % (item[0],))
break
else:
if is_sequence(item):
if item[0]==libitem:
warnings.warn("[2] libraries list contains %r with"
" no build_info" % (item[0],))
break
else:
if item==libitem:
return
libraries.append(item)
def _check_append_ext_library(libraries, (lib_name,build_info)):
for item in libraries:
if is_sequence(item):
if item[0]==lib_name:
if item[1] is build_info:
return
warnings.warn("[3] libraries list contains %r with"
" different build_info" % (lib_name,))
break
elif item==lib_name:
warnings.warn("[4] libraries list contains %r with"
" no build_info" % (lib_name,))
break
libraries.append((lib_name,build_info))
def build_a_library(self, build_info, lib_name, libraries):
# default compilers
compiler = self.compiler
fcompiler = self._f_compiler
sources = build_info.get('sources')
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(("in 'libraries' option (library '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % lib_name)
sources = list(sources)
c_sources, cxx_sources, f_sources, fmodule_sources \
= filter_sources(sources)
requiref90 = not not fmodule_sources or \
build_info.get('language', 'c')=='f90'
# save source type information so that build_ext can use it.
source_languages = []
if c_sources: source_languages.append('c')
if cxx_sources: source_languages.append('c++')
if requiref90: source_languages.append('f90')
elif f_sources: source_languages.append('f77')
build_info['source_languages'] = source_languages
lib_file = compiler.library_filename(lib_name,
output_dir=self.build_clib)
depends = sources + build_info.get('depends', [])
if not (self.force or newer_group(depends, lib_file, 'newer')):
log.debug("skipping '%s' library (up-to-date)", lib_name)
return
else:
log.info("building '%s' library", lib_name)
config_fc = build_info.get('config_fc', {})
if fcompiler is not None and config_fc:
log.info('using additional config_fc from setup script '\
'for fortran compiler: %s' \
% (config_fc,))
from numpy.distutils.fcompiler import new_fcompiler
fcompiler = new_fcompiler(compiler=fcompiler.compiler_type,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90=requiref90,
c_compiler=self.compiler)
if fcompiler is not None:
dist = self.distribution
base_config_fc = dist.get_option_dict('config_fc').copy()
base_config_fc.update(config_fc)
fcompiler.customize(base_config_fc)
# check availability of Fortran compilers
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("library %s has Fortran sources"\
" but no Fortran compiler found" % (lib_name))
if fcompiler is not None:
fcompiler.extra_f77_compile_args = build_info.get('extra_f77_compile_args') or []
fcompiler.extra_f90_compile_args = build_info.get('extra_f90_compile_args') or []
macros = build_info.get('macros')
include_dirs = build_info.get('include_dirs')
if include_dirs is None:
include_dirs = []
extra_postargs = build_info.get('extra_compiler_args') or []
include_dirs.extend(get_numpy_include_dirs())
# where compiled F90 module files are:
module_dirs = build_info.get('module_dirs') or []
module_build_dir = os.path.dirname(lib_file)
if requiref90: self.mkpath(module_build_dir)
if compiler.compiler_type=='msvc':
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
objects = []
if c_sources:
log.info("compiling C sources")
objects = compiler.compile(c_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
if cxx_sources:
log.info("compiling C++ sources")
cxx_compiler = compiler.cxx_compiler()
cxx_objects = cxx_compiler.compile(cxx_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
objects.extend(cxx_objects)
if f_sources or fmodule_sources:
extra_postargs = []
f_objects = []
if requiref90:
if fcompiler.module_dir_switch is None:
existing_modules = glob('*.mod')
extra_postargs += fcompiler.module_options(\
module_dirs, module_build_dir)
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
f_objects += fcompiler.compile(fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
if requiref90 and self._f_compiler.module_dir_switch is None:
# move new compiled F90 module files to module_build_dir
for f in glob('*.mod'):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f)==os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn('failed to move %r to %r' \
% (f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
else:
f_objects = []
objects.extend(f_objects)
# assume that default linker is suitable for
# linking Fortran object files
compiler.create_static_lib(objects, lib_name,
output_dir=self.build_clib,
debug=self.debug)
# fix library dependencies
clib_libraries = build_info.get('libraries', [])
for lname, binfo in libraries:
if lname in clib_libraries:
clib_libraries.extend(binfo[1].get('libraries', []))
if clib_libraries:
build_info['libraries'] = clib_libraries
def build_a_library(self, build_info, lib_name, libraries):
# default compilers
compiler = self.compiler
fcompiler = self._f_compiler
sources = build_info.get('sources')
if sources is None or not is_sequence(sources):
raise DistutilsSetupError(
("in 'libraries' option (library '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % lib_name)
sources = list(sources)
c_sources, cxx_sources, f_sources, fmodule_sources \
= filter_sources(sources)
requiref90 = not not fmodule_sources or \
build_info.get('language', 'c') == 'f90'
# save source type information so that build_ext can use it.
source_languages = []
if c_sources:
source_languages.append('c')
if cxx_sources:
source_languages.append('c++')
if requiref90:
source_languages.append('f90')
elif f_sources:
source_languages.append('f77')
build_info['source_languages'] = source_languages
lib_file = compiler.library_filename(lib_name,
output_dir=self.build_clib)
depends = sources + build_info.get('depends', [])
if not (self.force or newer_group(depends, lib_file, 'newer')):
log.debug("skipping '%s' library (up-to-date)", lib_name)
return
else:
log.info("building '%s' library", lib_name)
config_fc = build_info.get('config_fc', {})
if fcompiler is not None and config_fc:
log.info('using additional config_fc from setup script '
'for fortran compiler: %s' % (config_fc, ))
from numpy.distutils.fcompiler import new_fcompiler
fcompiler = new_fcompiler(compiler=fcompiler.compiler_type,
verbose=self.verbose,
dry_run=self.dry_run,
force=self.force,
requiref90=requiref90,
c_compiler=self.compiler)
if fcompiler is not None:
dist = self.distribution
base_config_fc = dist.get_option_dict('config_fc').copy()
base_config_fc.update(config_fc)
fcompiler.customize(base_config_fc)
# check availability of Fortran compilers
if (f_sources or fmodule_sources) and fcompiler is None:
raise DistutilsError("library %s has Fortran sources"
" but no Fortran compiler found" % (lib_name))
if fcompiler is not None:
fcompiler.extra_f77_compile_args = build_info.get(
'extra_f77_compile_args') or []
fcompiler.extra_f90_compile_args = build_info.get(
'extra_f90_compile_args') or []
macros = build_info.get('macros')
include_dirs = build_info.get('include_dirs')
if include_dirs is None:
include_dirs = []
extra_postargs = build_info.get('extra_compiler_args') or []
include_dirs.extend(get_numpy_include_dirs())
# where compiled F90 module files are:
module_dirs = build_info.get('module_dirs') or []
module_build_dir = os.path.dirname(lib_file)
if requiref90:
self.mkpath(module_build_dir)
if compiler.compiler_type == 'msvc':
# this hack works around the msvc compiler attributes
# problem, msvc uses its own convention :(
c_sources += cxx_sources
cxx_sources = []
objects = []
if c_sources:
log.info("compiling C sources")
objects = compiler.compile(c_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
if cxx_sources:
log.info("compiling C++ sources")
cxx_compiler = compiler.cxx_compiler()
cxx_objects = cxx_compiler.compile(cxx_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
objects.extend(cxx_objects)
if f_sources or fmodule_sources:
extra_postargs = []
f_objects = []
if requiref90:
if fcompiler.module_dir_switch is None:
existing_modules = glob('*.mod')
extra_postargs += fcompiler.module_options(
module_dirs, module_build_dir)
if fmodule_sources:
log.info("compiling Fortran 90 module sources")
f_objects += fcompiler.compile(fmodule_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
if requiref90 and self._f_compiler.module_dir_switch is None:
# move new compiled F90 module files to module_build_dir
for f in glob('*.mod'):
if f in existing_modules:
continue
t = os.path.join(module_build_dir, f)
if os.path.abspath(f) == os.path.abspath(t):
continue
if os.path.isfile(t):
os.remove(t)
try:
self.move_file(f, module_build_dir)
except DistutilsFileError:
log.warn('failed to move %r to %r' %
(f, module_build_dir))
if f_sources:
log.info("compiling Fortran sources")
f_objects += fcompiler.compile(f_sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug,
extra_postargs=extra_postargs)
else:
f_objects = []
if f_objects and not fcompiler.can_ccompiler_link(compiler):
# Default linker cannot link Fortran object files, and results
# need to be wrapped later. Instead of creating a real static
# library, just keep track of the object files.
listfn = os.path.join(self.build_clib, lib_name + '.fobjects')
with open(listfn, 'w') as f:
f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
listfn = os.path.join(self.build_clib, lib_name + '.cobjects')
with open(listfn, 'w') as f:
f.write("\n".join(os.path.abspath(obj) for obj in objects))
# create empty "library" file for dependency tracking
lib_fname = os.path.join(self.build_clib,
lib_name + compiler.static_lib_extension)
with open(lib_fname, 'wb') as f:
pass
else:
# assume that default linker is suitable for
# linking Fortran object files
objects.extend(f_objects)
compiler.create_static_lib(objects,
lib_name,
output_dir=self.build_clib,
debug=self.debug)
# fix library dependencies
clib_libraries = build_info.get('libraries', [])
for lname, binfo in libraries:
if lname in clib_libraries:
clib_libraries.extend(binfo.get('libraries', []))
if clib_libraries:
build_info['libraries'] = clib_libraries
