def parse_mysql_config_info(options, stdout):
log.debug("# stdout: {0}".format(stdout))
info = {}
for option, line in zip(options, stdout.split('\n')):
log.debug("# option: {0}".format(option))
log.debug("# line: {0}".format(line))
info[option] = line.strip()
ver = info['version']
if '-' in ver:
ver, _ = ver.split('-', 2)
info['version'] = tuple([int(v) for v in ver.split('.')[0:3]])
libs = shlex.split(info['libs'])
info['lib_dir'] = libs[0].replace('-L', '')
info['libs'] = [ lib.replace('-l', '') for lib in libs[1:] ]
log.debug("# info['libs']: ")
for lib in info['libs']:
log.debug("# {0}".format(lib))
libs = shlex.split(info['libs_r'])
info['lib_r_dir'] = libs[0].replace('-L', '')
info['libs_r'] = [ lib.replace('-l', '') for lib in libs[1:] ]
info['include'] = [x.strip() for x in info['include'].split('-I')[1:]]
return info
def get_externals(self):
cmd = ['svn', 'propget', 'svn:externals', self.path, '-R', '--xml']
code, lines = _run_command(cmd, encoding="utf-8")
if code:
log.debug("svn propget failed")
return []
return parse_externals_xml(lines, prefix=os.path.abspath(self.path))
def get_externals(self):
cmd = ["svn", "propget", "svn:externals", self.path, "-R", "--xml"]
code, lines = _run_command(cmd, encoding="utf-8")
if code:
log.debug("svn propget failed")
return []
return parse_externals_xml(lines, prefix=os.path.abspath(self.path))
def get_test_roots(self):
""" Find test directories, skipping nested dirs and anything marked
to skip under [pytest]:norecursedirs
"""
res = []
no_recurse = []
cfg = parse.get_pkg_cfg_parser()
if cfg.has_section('pytest') and \
cfg.has_option('pytest', 'norecursedirs'):
[no_recurse.extend(glob.glob(os.path.join(os.getcwd(), i)))
for i in cfg.get('pytest', 'norecursedirs').split()]
no_recurse = [os.path.abspath(i) for i in no_recurse]
test_dirs = []
for (dirpath, dirnames, _) in os.walk(os.getcwd()):
if CONFIG.test_dirname in dirnames:
test_dirs.append(os.path.join(dirpath, CONFIG.test_dirname))
test_dirs.sort(key=len)
for i in test_dirs:
try:
for j in res + no_recurse:
if i.startswith(j):
raise ValueError
except ValueError:
pass
else:
res.append(i)
log.debug("Test roots: {0}".format(res))
return res
def make_file(self, infiles, outfile, func, args,
exec_msg=None, skip_msg=None, level=1):
"""Special case of 'execute()' for operations that process one or
more input files and generate one output file. Works just like
'execute()', except the operation is skipped and a different
message printed if 'outfile' already exists and is newer than all
files listed in 'infiles'. If the command defined 'self.force',
and it is true, then the command is unconditionally run -- does no
timestamp checks.
"""
if exec_msg is None:
exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles))
if skip_msg is None:
skip_msg = "skipping %s (inputs unchanged)" % outfile
# Allow 'infiles' to be a single string
if isinstance(infiles, str):
infiles = (infiles,)
elif not isinstance(infiles, (list, tuple)):
raise TypeError(
"'infiles' must be a string, or a list or tuple of strings")
# If 'outfile' must be regenerated (either because it doesn't
# exist, is out-of-date, or the 'force' flag is true) then
# perform the action that presumably regenerates it
if self.force or dep_util.newer_group (infiles, outfile):
self.execute(func, args, exec_msg, level)
# Otherwise, print the "skip" message
else:
log.debug(skip_msg)
def query_vcvarsall(version, arch="x86"):
"""Launch vcvarsall.bat and read the settings from its environment
"""
vcvarsall = find_vcvarsall(version)
interesting = set(("include", "lib", "libpath", "path"))
result = {}
if vcvarsall is None:
raise DistutilsPlatformError("Unable to find vcvarsall.bat")
log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version)
popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = popen.communicate()
if popen.wait() != 0:
raise DistutilsPlatformError(stderr.decode("mbcs"))
stdout = stdout.decode("mbcs")
for line in stdout.split("\n"):
line = Reg.convert_mbcs(line)
if '=' not in line:
continue
line = line.strip()
key, value = line.split('=', 1)
key = key.lower()
if key in interesting:
if value.endswith(os.pathsep):
value = value[:-1]
result[key] = removeDuplicates(value)
if len(result) != len(interesting):
raise ValueError(str(list(result.keys())))
return result
def create_static_lib(self, objects, output_libname,
output_dir=None, debug=0, target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
output_filename = \
self.library_filename(output_libname, output_dir=output_dir)
if self._need_link(objects, output_filename):
self.mkpath(os.path.dirname(output_filename))
self.spawn(self.archiver +
[output_filename] +
objects + self.objects)
# Not many Unices required ranlib anymore -- SunOS 4.x is, I
# think the only major Unix that does. Maybe we need some
# platform intelligence here to skip ranlib if it's not
# needed -- or maybe Python's configure script took care of
# it for us, hence the check for leading colon.
if self.ranlib:
try:
self.spawn(self.ranlib + [output_filename])
except DistutilsExecError as msg:
raise LibError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def run_cython(source, depends=(), includes=(),
destdir_c=None, destdir_h=None,
wdir=None, force=False, VERSION=None):
from glob import glob
from distutils import log
from distutils import dep_util
from distutils.errors import DistutilsError
target = os.path.splitext(source)[0]+'.c'
cwd = os.getcwd()
try:
if wdir: os.chdir(wdir)
alldeps = [source]
for dep in depends:
alldeps += glob(dep)
if not (force or dep_util.newer_group(alldeps, target)):
log.debug("skipping '%s' -> '%s' (up-to-date)",
source, target)
return
finally:
os.chdir(cwd)
if not chk_cython(VERSION):
raise DistutilsError("requires Cython>=%s" % VERSION)
log.info("cythonizing '%s' -> '%s'", source, target)
from cythonize import cythonize
err = cythonize(source,
includes=includes,
destdir_c=destdir_c,
destdir_h=destdir_h,
wdir=wdir)
if err:
raise DistutilsError(
"Cython failure: '%s' -> '%s'" % (source, target))
def find_vcvarsall(version):
"""Find the vcvarsall.bat file
At first it tries to find the productdir of VS 2008 in the registry. If
that fails it falls back to the VS90COMNTOOLS env var.
"""
vsbase = VS_BASE % version
try:
productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
"productdir")
except KeyError:
log.debug("Unable to find productdir in registry")
productdir = None
if not productdir or not os.path.isdir(productdir):
toolskey = "VS%0.f0COMNTOOLS" % version
toolsdir = os.environ.get(toolskey, None)
if toolsdir and os.path.isdir(toolsdir):
productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
productdir = os.path.abspath(productdir)
if not os.path.isdir(productdir):
log.debug("%s is not a valid directory" % productdir)
return None
else:
log.debug("Env var %s is not set or invalid" % toolskey)
if not productdir:
log.debug("No productdir found")
return None
vcvarsall = os.path.join(productdir, "vcvarsall.bat")
if os.path.isfile(vcvarsall):
return vcvarsall
log.debug("Unable to find vcvarsall.bat")
return None
def get_extensions():
med_sources = [str(os.path.join(UTIL_DIR, "median_utils.pyx")),
str(os.path.join(UTIL_DIR, "quick_select.c"))]
include_dirs = ['numpy', UTIL_DIR]
libraries = []
ext_med = Extension(name=str('banzai.utils.median_utils'),
sources=med_sources,
include_dirs=include_dirs,
libraries=libraries,
language="c",
extra_compile_args=['-g', '-O3', '-funroll-loops', '-ffast-math'])
has_openmp, outputs = check_openmp()
if has_openmp:
if setup_helpers.get_compiler_option() == 'msvc':
ext_med.extra_compile_args.append('-openmp')
else:
ext_med.extra_compile_args.append('-fopenmp')
ext_med.extra_link_args = ['-g', '-fopenmp']
else:
log.warn('OpenMP was not found. '
'banzai will be compiled without OpenMP. '
'(Use the "-v" option of setup.py for more details.)')
log.debug(('(Start of OpenMP info)\n'
'compiler stdout:\n{0}\n'
'compiler stderr:\n{1}\n'
'(End of OpenMP info)').format(*outputs))
return [ext_med]
def run(self):
# remove the build/temp.<plat> directory (unless it's already
# gone)
if os.path.exists(self.build_temp):
remove_tree(self.build_temp, dry_run=self.dry_run)
else:
log.debug("'%s' does not exist -- can't clean it",
self.build_temp)
if self.all:
# remove build directories
for directory in (self.build_lib,
self.bdist_base,
self.build_scripts):
if os.path.exists(directory):
remove_tree(directory, dry_run=self.dry_run)
else:
log.warn("'%s' does not exist -- can't clean it",
directory)
# just for the heck of it, try to remove the base build directory:
# we might have emptied it right now, but if not we don't care
if not self.dry_run:
try:
os.rmdir(self.build_base)
log.info("removing '%s'", self.build_base)
except OSError:
pass
def build_extension(self, ext):
sources = ext.sources
if sources is None or type(sources) not in (ListType, TupleType):
raise DistutilsSetupError, ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + 'a list of source filenames') % ext.name
sources = list(sources)
ext_path = self.get_ext_fullpath(ext.name)
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_path, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
sources = self.swig_sources(sources, ext)
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef,))
objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, include_dirs=ext.include_dirs, debug=self.debug, extra_postargs=extra_args, depends=ext.depends)
self._built_objects = objects[:]
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
language = ext.language or self.compiler.detect_language(sources)
self.compiler.link_shared_object(objects, ext_path, libraries=self.get_libraries(ext), library_dirs=ext.library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=language)
return
def chmod(path, mode):
from distutils import log
log.debug("changing mode of %s to %o", path, mode)
try:
_chmod(path, mode)
except os.error, e:
log.debug("chmod failed: %s", e)
def get_test_roots(self):
""" Find test directories, skipping nested dirs and anything marked
to skip under [pytest]:norecursedirs
"""
from path import path
res = []
no_recurse = []
cfg = config.get_pkg_cfg_parser()
if cfg.has_section('pytest') and \
cfg.has_option('pytest', 'norecursedirs'):
[no_recurse.extend(path.getcwd().glob(i)) for i in
cfg.get('pytest', 'norecursedirs').split()]
no_recurse = [i.abspath() for i in no_recurse]
test_dirs = [i for i in
path.getcwd().walkdirs(CONFIG.test_dirname)]
test_dirs.sort(key=len)
for i in test_dirs:
try:
for j in res + no_recurse:
if i.startswith(j):
raise ValueError
except ValueError:
pass
else:
res.append(i)
log.debug("Test roots: {0}".format(res))
return res
def test_non_ascii(self):
# Issues #8663, #34421: test that non-encodable text is escaped with
# backslashreplace error handler and encodable non-ASCII text is
# output as is.
for errors in ('strict', 'backslashreplace', 'surrogateescape',
'replace', 'ignore'):
with self.subTest(errors=errors):
stdout = io.TextIOWrapper(io.BytesIO(),
encoding='cp437', errors=errors)
stderr = io.TextIOWrapper(io.BytesIO(),
encoding='cp437', errors=errors)
old_threshold = log.set_threshold(log.DEBUG)
try:
with swap_attr(sys, 'stdout', stdout), \
swap_attr(sys, 'stderr', stderr):
log.debug('Dεbug\tMėssãge')
log.fatal('Fαtal\tÈrrōr')
finally:
log.set_threshold(old_threshold)
stdout.seek(0)
self.assertEqual(stdout.read().rstrip(),
'Dεbug\tM?ss?ge' if errors == 'replace' else
'Dεbug\tMssge' if errors == 'ignore' else
'Dεbug\tM\\u0117ss\\xe3ge')
stderr.seek(0)
self.assertEqual(stderr.read().rstrip(),
'Fαtal\t?rr?r' if errors == 'replace' else
'Fαtal\trrr' if errors == 'ignore' else
'Fαtal\t\\xc8rr\\u014dr')
def _find_vc2015():
try:
key = winreg.OpenKeyEx(
winreg.HKEY_LOCAL_MACHINE,
r"Software\Microsoft\VisualStudio\SxS\VC7",
access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
)
except OSError:
log.debug("Visual C++ is not registered")
return None, None
best_version = 0
best_dir = None
with key:
for i in count():
try:
v, vc_dir, vt = winreg.EnumValue(key, i)
except OSError:
break
if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir):
try:
version = int(float(v))
except (ValueError, TypeError):
continue
if version >= 14 and version > best_version:
best_version, best_dir = version, vc_dir
return best_version, best_dir
def _writeclientdoc(doc, thing, forceload=0):
"""Write HTML documentation to a file in the current directory.
"""
docmodule = pydoc.HTMLDoc.docmodule
def strongarm(self, object, name=None, mod=None, *ignored):
result = docmodule(self, object, name, mod, *ignored)
# Grab all the aliases to pyclasses and create links.
nonmembers = []
push = nonmembers.append
for k,v in inspect.getmembers(object, inspect.isclass):
if inspect.getmodule(v) is not object and getattr(v,'typecode',None) is not None:
push('<a href="%s.html">%s</a>: pyclass alias<br/>' %(v.__name__,k))
result += self.bigsection('Aliases', '#ffffff', '#eeaa77', ''.join(nonmembers))
return result
pydoc.HTMLDoc.docmodule = strongarm
try:
object, name = pydoc.resolve(thing, forceload)
page = pydoc.html.page(pydoc.describe(object), pydoc.html.document(object, name))
name = os.path.join(doc, name + '.html')
file = open(name, 'w')
file.write(page)
file.close()
except (ImportError, pydoc.ErrorDuringImport), value:
log.debug(str(value))
def run(self):
self.run_command('build')
build_lib = self.get_finalized_command('build').build_lib
old_pythonpath = None
try:
if 'PYTHONPATH' in os.environ:
old_pythonpath = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = build_lib + ':' + old_pythonpath
else:
os.environ['PYTHONPATH'] = build_lib
python_exec = os.path.normpath(sys.executable)
log.info("running unit tests...")
try:
cmd = command(python_exec, "-m", "netsa.dist.run_unit_tests",
"-v", *self.distribution.netsa_unit_test_modules)
log.debug("PYTHONPATH=%s %s", os.environ['PYTHONPATH'], cmd)
run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
except:
log.info("running unit tests... failed")
sys.exit(-1)
log.info("running unit tests... success")
finally:
if old_pythonpath:
os.environ['PYTHONPATH'] = old_pythonpath
else:
del os.environ['PYTHONPATH']
def run(self):
self.run_command('build')
build_lib = self.get_finalized_command('build').build_lib
old_pythonpath = None
try:
if 'PYTHONPATH' in os.environ:
old_pythonpath = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = build_lib + ':' + old_pythonpath
else:
os.environ['PYTHONPATH'] = build_lib
python_exec = os.path.normpath(sys.executable)
log.info("running other tests...")
errors = False
for test_mod in self.distribution.netsa_other_test_modules:
try:
cmd = command(python_exec, "-m", test_mod,
self.distribution.source_dir)
log.debug("PYTHONPATH=%s %s",
os.environ['PYTHONPATH'], cmd)
run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
except:
errors = True
if errors:
log.info("running other tests... failed")
sys.exit(-1)
log.info("running other tests... success")
finally:
if old_pythonpath:
os.environ['PYTHONPATH'] = old_pythonpath
else:
del os.environ['PYTHONPATH']
def run(self):
if self.all:
for (filename, template) in self.distribution.netsa_version_files:
log.info("removing %r", filename)
if not self.dry_run:
try:
os.unlink(filename)
except:
log.debug("%r doesn't exist -- can't clean it",
filename)
html_dir = self.get_finalized_command('gen_doc_html').gen_doc_html
if os.path.exists(html_dir):
remove_tree(html_dir, dry_run=self.dry_run)
pdf_file = self.get_finalized_command('gen_doc_pdf').gen_doc_pdf
log.info("removing %r", pdf_file)
if not self.dry_run:
try:
os.unlink(pdf_file)
except:
log.debug("%r doesn't exist -- can't clean it",
pdf_file)
latex_dir = self.get_finalized_command('gen_doc_pdf').gen_doc_latex
extra_dir = self.get_finalized_command('gen_doc_pdf').gen_doc_extra
if os.path.exists(latex_dir):
remove_tree(latex_dir, dry_run=self.dry_run)
if os.path.exists(extra_dir):
remove_tree(extra_dir, dry_run=self.dry_run)
clean.run(self)
def run(self):
self.run_command('build')
build_lib = self.get_finalized_command('build').build_lib
if self.distribution.netsa_doc_dir:
self.mkpath(self.gen_doc_extra)
gen_doc_config(self.distribution, self.gen_doc_extra)
old_pythonpath = None
try:
if 'PYTHONPATH' in os.environ:
old_pythonpath = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = (
self.gen_doc_extra + ':' + build_lib +
':' + old_pythonpath)
else:
os.environ['PYTHONPATH'] = (
self.gen_doc_extra + ":" + build_lib)
cmd = command("sphinx-build",
"-b", "html",
"-d", self.gen_doc_extra,
os.path.join(self.distribution.source_dir,
self.distribution.netsa_doc_dir),
self.gen_doc_html)
log.info("generating HTML documentation with Sphinx")
log.debug("PYTHONPATH=%s %s", os.environ['PYTHONPATH'], cmd)
run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
finally:
if old_pythonpath:
os.environ['PYTHONPATH'] = old_pythonpath
else:
del os.environ['PYTHONPATH']
def make_archive(base_name, format, root_dir = None, base_dir = None, verbose = 0, dry_run = 0, owner = None, group = None):
save_cwd = os.getcwd()
if root_dir is not None:
log.debug("changing into '%s'", root_dir)
base_name = os.path.abspath(base_name)
if not dry_run:
os.chdir(root_dir)
if base_dir is None:
base_dir = os.curdir
kwargs = {'dry_run': dry_run}
try:
format_info = ARCHIVE_FORMATS[format]
except KeyError:
raise ValueError, "unknown archive format '%s'" % format
func = format_info[0]
for arg, val in format_info[1]:
kwargs[arg] = val
if format != 'zip':
kwargs['owner'] = owner
kwargs['group'] = group
try:
filename = func(base_name, base_dir, **kwargs)
finally:
if root_dir is not None:
log.debug("changing back to '%s'", save_cwd)
os.chdir(save_cwd)
return filename
def _run_command(self, cmd, env=None):
log.debug('running [%s]' % (' '.join(cmd), ))
try:
return check_output(cmd, cwd=self.work_path, env=env)
except Exception:
log.error('command failed [%s] via [%s]' % (' '.join(cmd), self.work_path, ))
raise
def run(self):
if self.all:
if os.path.exists(self.build_doc):
remove_tree(self.build_doc, dry_run=self.dry_run)
else:
log.debug("%s doesn't exist -- can't clean it", self.build_doc)
_clean.run(self)
def __build_java_lib(self, jlib):
log.info("Building java code for hadoop-%s" % jlib.hadoop_vinfo)
compile_cmd = "javac"
if jlib.classpath:
compile_cmd += " -classpath %s" % jlib.classpath
else:
log.warn("WARNING: could not set classpath, java code may not compile")
class_dir = os.path.join(self.build_temp, "pipes-%s" % jlib.hadoop_vinfo)
package_path = os.path.join(self.build_lib, "pydoop", jlib.jar_name)
if not os.path.exists(class_dir):
os.mkdir(class_dir)
compile_cmd += " -d '%s'" % class_dir
log.info("Compiling Java classes")
for f in jlib.java_files:
compile_cmd += " %s" % f
log.debug("Command: %s", compile_cmd)
ret = os.system(compile_cmd)
if ret:
raise DistutilsSetupError(
"Error compiling java component. Command: %s" % compile_cmd
)
package_cmd = "jar -cf %(package_path)s -C %(class_dir)s ./it" % {
'package_path': package_path, 'class_dir': class_dir
}
log.info("Packaging Java classes")
log.debug("Command: %s", package_cmd)
ret = os.system(package_cmd)
if ret:
raise DistutilsSetupError(
"Error packaging java component. Command: %s" % package_cmd
)
def zap_pyfiles(self):
log.info("Removing .py files from temporary directory")
for base, dirs, files in walk_egg(self.bdist_dir):
for name in files:
path = os.path.join(base, name)
if name.endswith('.py'):
log.debug("Deleting %s", path)
os.unlink(path)
if base.endswith('__pycache__'):
path_old = path
pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'
m = re.match(pattern, name)
path_new = os.path.join(
base, os.pardir, m.group('name') + '.pyc')
log.info(
"Renaming file from [%s] to [%s]"
% (path_old, path_new))
try:
os.remove(path_new)
except OSError:
pass
os.rename(path_old, path_new)
def run (self):
# Obviously have to build before we can install
if not self.skip_build:
self.run_command('build')
# Run all sub-commands (at least those that need to be run)
for cmd_name in self.get_sub_commands():
self.run_command(cmd_name)
if self.path_file:
self.create_path_file()
# write list of installed files, if requested.
if self.record:
outputs = self.get_outputs()
if self.root: # strip any package prefix
root_len = len(self.root)
for counter in xrange(len(outputs)):
outputs[counter] = outputs[counter][root_len:]
self.execute(write_file,
(self.record, outputs),
"writing list of installed files to '%s'" %
self.record)
sys_path = map(os.path.normpath, sys.path)
sys_path = map(os.path.normcase, sys_path)
install_lib = os.path.normcase(os.path.normpath(self.install_lib))
if (self.warn_dir and
not (self.path_file and self.install_path_file) and
install_lib not in sys_path):
log.debug(("modules installed to '%s', which is not in "
"Python's module search path (sys.path) -- "
"you'll have to change the search path yourself"),
self.install_lib)
def run(self):
if not self.source_dir:
log.error('no source directory specified')
return
for root, dirs, files in os.walk(self.source_dir):
targ_dir = os.path.join(
root.replace(self.source_dir, self.build_dir), 'LC_MESSAGES')
for name in files:
base, ext = os.path.splitext(name)
if ext.lower() != '.po':
continue
src = os.path.join(root, name)
targ = os.path.join(targ_dir, base + '.mo')
if os.path.exists(targ):
targ_date = os.stat(targ)[8]
else:
targ_date = 0
src_date = os.stat(src)[8]
if targ_date > src_date:
log.debug('skipping %s', targ)
continue
self.mkpath(targ_dir)
log.info('compiling %s to %s', src, targ)
if not self.dry_run:
subprocess.check_call(
['msgfmt', '--use-fuzzy',
'--output-file=%s' % targ, src])
def find_sources(self):
"""Finds the sources as default and then drop the cruft."""
vsc_egg_info.find_sources(self)
for fn in remove_extra_bdist_rpm_files():
log.debug("removing %s from source list" % (fn))
if fn in self.filelist.files:
self.filelist.files.remove(fn)
def _get_dlls(self):
# return a list of of (FQ-in-name, relative-out-name) tuples.
ret = []
# the apr binaries.
apr_bins = [libname + ".dll" for libname in extra_libs
if libname.startswith("libapr")]
if get_svn_version() >= (1,5,0):
# Since 1.5.0 these libraries became shared
apr_bins += """libsvn_client-1.dll libsvn_delta-1.dll libsvn_diff-1.dll
libsvn_fs-1.dll libsvn_ra-1.dll libsvn_repos-1.dll
libsvn_subr-1.dll libsvn_wc-1.dll libsasl.dll""".split()
if get_svn_version() >= (1,7,0):
apr_bins += ["libdb48.dll"]
else:
apr_bins += ["libdb44.dll"]
apr_bins += """intl3_svn.dll libeay32.dll ssleay32.dll""".split()
look_dirs = os.environ.get("PATH","").split(os.pathsep)
look_dirs.insert(0, os.path.join(os.environ["SVN_DEV"], "bin"))
target = os.path.abspath(os.path.join(self.build_lib, 'subvertpy'))
for bin in apr_bins:
for look in look_dirs:
f = os.path.join(look, bin)
if os.path.isfile(f):
ret.append((f, target))
break
else:
log.warn("Could not find required DLL %r to include", bin)
log.debug("(looked in %s)", look_dirs)
return ret
def copy_scripts(self):
"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
self.mkpath(self.build_dir)
outfiles = []
for script in self.scripts:
adjust = 0
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
if not self.force and not newer(script, outfile):
log.debug("not copying %s (up-to-date)", script)
continue
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "r")
except IOError:
if not self.dry_run:
raise
f = None
else:
first_line = f.readline()
if not first_line:
self.warn("%s is an empty file (skipping)" % script)
continue
match = first_line_re.match(first_line)
if match:
adjust = 1
post_interp = match.group(1) or ''
if adjust:
log.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
outf = open(outfile, "w")
if not sysconfig.python_build:
outf.write("#!%s%s\n" % (self.executable, post_interp))
else:
outf.write("#!%s%s\n" % (os.path.join(
sysconfig.get_config_var("BINDIR"),
"python" + sysconfig.get_config_var("VERSION") +
sysconfig.get_config_var("EXE")), post_interp))
outf.writelines(f.readlines())
outf.close()
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
log.info("changing mode of %s", file)
else:
oldmode = os.stat(file)[ST_MODE] & 07777
newmode = (oldmode | 0555) & 07777
if newmode != oldmode:
log.info("changing mode of %s from %o to %o", file,
oldmode, newmode)
os.chmod(file, newmode)
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
# XXX this ignores 'build_temp'! should follow the lead of
# msvccompiler.py
(objects, output_dir) = self._fix_object_args(objects, output_dir)
(libraries, library_dirs, runtime_library_dirs) = \
self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)
if runtime_library_dirs:
log.warn("I don't know what to do with 'runtime_library_dirs': %s",
str(runtime_library_dirs))
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
# Figure out linker args based on type of target.
if target_desc == CCompiler.EXECUTABLE:
startup_obj = 'c0w32'
if debug:
ld_args = self.ldflags_exe_debug[:]
else:
ld_args = self.ldflags_exe[:]
else:
startup_obj = 'c0d32'
if debug:
ld_args = self.ldflags_shared_debug[:]
else:
ld_args = self.ldflags_shared[:]
# Create a temporary exports file for use by the linker
if export_symbols is None:
def_file = ''
else:
head, tail = os.path.split(output_filename)
modname, ext = os.path.splitext(tail)
temp_dir = os.path.dirname(
objects[0]) # preserve tree structure
def_file = os.path.join(temp_dir, '%s.def' % modname)
contents = ['EXPORTS']
for sym in (export_symbols or []):
contents.append(' %s=_%s' % (sym, sym))
self.execute(write_file, (def_file, contents),
"writing %s" % def_file)
# Borland C++ has problems with '/' in paths
objects2 = map(os.path.normpath, objects)
# split objects in .obj and .res files
# Borland C++ needs them at different positions in the command line
objects = [startup_obj]
resources = []
for file in objects2:
(base, ext) = os.path.splitext(os.path.normcase(file))
if ext == '.res':
resources.append(file)
else:
objects.append(file)
for l in library_dirs:
ld_args.append("/L%s" % os.path.normpath(l))
ld_args.append("/L.") # we sometimes use relative paths
# list of object files
ld_args.extend(objects)
# XXX the command-line syntax for Borland C++ is a bit wonky;
# certain filenames are jammed together in one big string, but
# comma-delimited. This doesn't mesh too well with the
# Unix-centric attitude (with a DOS/Windows quoting hack) of
# 'spawn()', so constructing the argument list is a bit
# awkward. Note that doing the obvious thing and jamming all
# the filenames and commas into one argument would be wrong,
# because 'spawn()' would quote any filenames with spaces in
# them. Arghghh!. Apparently it works fine as coded...
# name of dll/exe file
ld_args.extend([',', output_filename])
# no map file and start libraries
ld_args.append(',,')
for lib in libraries:
# see if we find it and if there is a bcpp specific lib
# (xxx_bcpp.lib)
libfile = self.find_library_file(library_dirs, lib, debug)
if libfile is None:
ld_args.append(lib)
# probably a BCPP internal library -- don't warn
else:
# full name which prefers bcpp_xxx.lib over xxx.lib
ld_args.append(libfile)
# some default libraries
ld_args.append('import32')
ld_args.append('cw32mt')
# def file for export symbols
ld_args.extend([',', def_file])
# add resource files
ld_args.append(',')
ld_args.extend(resources)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def build_extension(self, ext):
sources = ext.sources
if sources is None or type(sources) not in (ListType, TupleType):
raise DistutilsSetupError, \
("in 'ext_modules' option (extension '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % ext.name
sources = list(sources)
fullname = self.get_ext_fullname(ext.name)
if self.inplace:
# ignore build-lib -- put the compiled extension into
# the source tree along with pure Python modules
modpath = string.split(fullname, '.')
package = string.join(modpath[0:-1], '.')
base = modpath[-1]
build_py = self.get_finalized_command('build_py')
package_dir = build_py.get_package_dir(package)
ext_filename = os.path.join(package_dir,
self.get_ext_filename(base))
else:
ext_filename = os.path.join(self.build_lib,
self.get_ext_filename(fullname))
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_filename, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
# First, scan the sources for SWIG definition files (.i), run
# SWIG on 'em to create .c files, and modify the sources list
# accordingly.
sources = self.swig_sources(sources, ext)
# Next, compile the source code to object files.
# XXX not honouring 'define_macros' or 'undef_macros' -- the
# CCompiler API needs to change to accommodate this, and I
# want to do one thing at a time!
# Two possible sources for extra compiler arguments:
# - 'extra_compile_args' in Extension object
# - CFLAGS environment variable (not particularly
# elegant, but people seem to expect it and I
# guess it's useful)
# The environment variable should take precedence, and
# any sensible compiler will give precedence to later
# command line args. Hence we combine them in order:
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef, ))
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=ext.include_dirs,
debug=self.debug,
extra_postargs=extra_args,
depends=ext.depends)
# XXX -- this is a Vile HACK!
#
# The setup.py script for Python on Unix needs to be able to
# get this list so it can perform all the clean up needed to
# avoid keeping object files around when cleaning out a failed
# build of an extension module. Since Distutils does not
# track dependencies, we have to get rid of intermediates to
# ensure all the intermediates will be properly re-built.
#
self._built_objects = objects[:]
# Now link the object files together into a "shared object" --
# of course, first we have to figure out all the other things
# that go into the mix.
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
# Detect target language, if not provided
language = ext.language or self.compiler.detect_language(sources)
self.compiler.link_shared_object(
objects,
ext_filename,
libraries=self.get_libraries(ext),
library_dirs=ext.library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=language)
def link_osx(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
"""Link the objects."""
# use separate copies, so we can modify the lists
libraries = copy.copy(libraries or [])
libraries.extend(self.dll_libraries)
# Copy of unix link so we can remove the darwin test
#UnixCCompiler.link(self, target_desc, objects, output_filename,
# output_dir, libraries, library_dirs, runtime_library_dirs,
# None, # export_symbols, we do this in our def-file
# debug, extra_preargs, extra_postargs, build_temp, target_lang)
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = \
self._fix_lib_args(libraries, library_dirs, runtime_library_dirs)
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
#if type(output_dir) not in (StringType, NoneType):
# raise TypeError("'output_dir' must be a string or None")
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
ld_args = (objects + self.objects + lib_opts +
['-o', output_filename])
if debug:
ld_args[:0] = ['-g']
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
if target_desc == CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i = i + 1
linker[i] = self.compiler_cxx[i]
#if sys.platform == 'darwin':
# linker = _osx_support.compiler_fixup(linker, ld_args)
# ld_args = ['-arch', 'x86_64'] + ld_args
self.spawn(linker + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def build_extension(self, ext):
sources = ext.sources
if sources is None or not isinstance(sources, (list, tuple)):
raise DistutilsSetupError(
"in 'ext_modules' option (extension '%s'), "
"'sources' must be present and must be "
"a list of source filenames" % ext.name)
sources = list(sources)
ext_path = self.get_ext_fullpath(ext.name)
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_path, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
# First, scan the sources for SWIG definition files (.i), run
# SWIG on 'em to create .c files, and modify the sources list
# accordingly.
sources = self.swig_sources(sources, ext)
# Next, compile the source code to object files.
# XXX not honouring 'define_macros' or 'undef_macros' -- the
# CCompiler API needs to change to accommodate this, and I
# want to do one thing at a time!
# Two possible sources for extra compiler arguments:
# - 'extra_compile_args' in Extension object
# - CFLAGS environment variable (not particularly
# elegant, but people seem to expect it and I
# guess it's useful)
# The environment variable should take precedence, and
# any sensible compiler will give precedence to later
# command line args. Hence we combine them in order:
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef, ))
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=ext.include_dirs,
debug=self.debug,
extra_postargs=extra_args,
depends=ext.depends)
# XXX outdated variable, kept here in case third-part code
# needs it.
self._built_objects = objects[:]
# Now link the object files together into a "shared object" --
# of course, first we have to figure out all the other things
# that go into the mix.
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
# Detect target language, if not provided
language = ext.language or self.compiler.detect_language(sources)
self.compiler.link_shared_object(
objects,
ext_path,
libraries=self.get_libraries(ext),
library_dirs=ext.library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=language)
def link(self, target_desc, objects,
output_filename, output_dir=None, libraries=None,
library_dirs=None, runtime_library_dirs=None,
export_symbols=None, debug=0, extra_preargs=None,
extra_postargs=None, build_temp=None, target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
libraries, library_dirs, runtime_library_dirs = fixed_args
# filter out standard library paths, which are not explicitely needed
# for linking
system_libdirs = ['/lib', '/lib64', '/usr/lib', '/usr/lib64']
multiarch = sysconfig.get_config_var("MULTIARCH")
if multiarch:
system_libdirs.extend(['/lib/%s' % multiarch, '/usr/lib/%s' % multiarch])
library_dirs = [dir for dir in library_dirs
if not dir in system_libdirs]
runtime_library_dirs = [dir for dir in runtime_library_dirs
if not dir in system_libdirs]
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
if not isinstance(output_dir, (str, type(None))):
raise TypeError("'output_dir' must be a string or None")
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
ld_args = (objects + self.objects +
lib_opts + ['-o', output_filename])
if debug:
ld_args[:0] = ['-g']
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
if target_desc == CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i += 1
linker[i] = self.compiler_cxx[i]
if sys.platform == 'darwin':
linker = _osx_support.compiler_fixup(linker, ld_args)
self.spawn(linker + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def announce(self, msg, level=1):
log.debug(msg)
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
if not self.initialized:
self.initialize()
(objects, output_dir) = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
(libraries, library_dirs, runtime_library_dirs) = fixed_args
if runtime_library_dirs:
self.warn ("I don't know what to do with 'runtime_library_dirs': "
+ str (runtime_library_dirs))
lib_opts = gen_lib_options(self,
library_dirs, runtime_library_dirs,
libraries)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
if debug:
ldflags = self.ldflags_shared_debug[1:]
else:
ldflags = self.ldflags_shared[1:]
else:
if debug:
ldflags = self.ldflags_shared_debug
else:
ldflags = self.ldflags_shared
export_opts = []
for sym in (export_symbols or []):
export_opts.append("/EXPORT:" + sym)
ld_args = (ldflags + lib_opts + export_opts +
objects + ['/OUT:' + output_filename])
# The MSVC linker generates .lib and .exp files, which cannot be
# suppressed by any linker switches. The .lib files may even be
# needed! Make sure they are generated in the temporary build
# directory. Since they have different names for debug and release
# builds, they can go into the same directory.
if export_symbols is not None:
(dll_name, dll_ext) = os.path.splitext(
os.path.basename(output_filename))
implib_file = os.path.join(
os.path.dirname(objects[0]),
self.library_filename(dll_name))
ld_args.append ('/IMPLIB:' + implib_file)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def byte_compile(
py_files,
optimize=0,
force=0,
prefix=None,
base_dir=None,
verbose=1,
dry_run=0,
direct=None,
):
"""Byte-compile a collection of Python source files to .pyc
files in a __pycache__ subdirectory. 'py_files' is a list
of files to compile; any files that don't end in ".py" are silently
skipped. 'optimize' must be one of the following:
0 - don't optimize
1 - normal optimization (like "python -O")
2 - extra optimization (like "python -OO")
If 'force' is true, all files are recompiled regardless of
timestamps.
The source filename encoded in each bytecode file defaults to the
filenames listed in 'py_files'; you can modify these with 'prefix' and
'basedir'. 'prefix' is a string that will be stripped off of each
source filename, and 'base_dir' is a directory name that will be
prepended (after 'prefix' is stripped). You can supply either or both
(or neither) of 'prefix' and 'base_dir', as you wish.
If 'dry_run' is true, doesn't actually do anything that would
affect the filesystem.
Byte-compilation is either done directly in this interpreter process
with the standard py_compile module, or indirectly by writing a
temporary script and executing it. Normally, you should let
'byte_compile()' figure out to use direct compilation or not (see
the source for details). The 'direct' flag is used by the script
generated in indirect mode; unless you know what you're doing, leave
it set to None.
"""
# nothing is done if sys.dont_write_bytecode is True
if sys.dont_write_bytecode:
raise DistutilsByteCompileError('byte-compiling is disabled.')
# First, if the caller didn't force us into direct or indirect mode,
# figure out which mode we should be in. We take a conservative
# approach: choose direct mode *only* if the current interpreter is
# in debug mode and optimize is 0. If we're not in debug mode (-O
# or -OO), we don't know which level of optimization this
# interpreter is running with, so we can't do direct
# byte-compilation and be certain that it's the right thing. Thus,
# always compile indirectly if the current interpreter is in either
# optimize mode, or if either optimization level was requested by
# the caller.
if direct is None:
direct = __debug__ and optimize == 0
# "Indirect" byte-compilation: write a temporary script and then
# run it with the appropriate flags.
if not direct:
try:
from tempfile import mkstemp
(script_fd, script_name) = mkstemp(".py")
except ImportError:
from tempfile import mktemp
(script_fd, script_name) = None, mktemp(".py")
log.info("writing byte-compilation script '%s'", script_name)
if not dry_run:
if script_fd is not None:
script = os.fdopen(script_fd, "w")
else:
script = open(script_name, "w")
with script:
script.write("""\
from distutils.util import byte_compile
files = [
""")
# XXX would be nice to write absolute filenames, just for
# safety's sake (script should be more robust in the face of
# chdir'ing before running it). But this requires abspath'ing
# 'prefix' as well, and that breaks the hack in build_lib's
# 'byte_compile()' method that carefully tacks on a trailing
# slash (os.sep really) to make sure the prefix here is "just
# right". This whole prefix business is rather delicate -- the
# problem is that it's really a directory, but I'm treating it
# as a dumb string, so trailing slashes and so forth matter.
script.write(",\n".join(map(repr, py_files)) + "]\n")
script.write("""
byte_compile(files, optimize=%r, force=%r,
prefix=%r, base_dir=%r,
verbose=%r, dry_run=0,
direct=1)
""" % (optimize, force, prefix, base_dir, verbose))
cmd = [sys.executable]
cmd.extend(subprocess._optim_args_from_interpreter_flags())
cmd.append(script_name)
spawn(cmd, dry_run=dry_run)
execute(os.remove, (script_name, ),
"removing %s" % script_name,
dry_run=dry_run)
# "Direct" byte-compilation: use the py_compile module to compile
# right here, right now. Note that the script generated in indirect
# mode simply calls 'byte_compile()' in direct mode, a weird sort of
# cross-process recursion. Hey, it works!
else:
from py_compile import compile
for file in py_files:
if file[-3:] != ".py":
# This lets us be lazy and not filter filenames in
# the "install_lib" command.
continue
# Terminology from the py_compile module:
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
if optimize >= 0:
opt = '' if optimize == 0 else optimize
cfile = importlib.util.cache_from_source(file,
optimization=opt)
else:
cfile = importlib.util.cache_from_source(file)
dfile = file
if prefix:
if file[:len(prefix)] != prefix:
raise ValueError(
"invalid prefix: filename %r doesn't start with %r" %
(file, prefix))
dfile = dfile[len(prefix):]
if base_dir:
dfile = os.path.join(base_dir, dfile)
cfile_base = os.path.basename(cfile)
if direct:
if force or newer(file, cfile):
log.info("byte-compiling %s to %s", file, cfile_base)
if not dry_run:
compile(file, cfile, dfile)
else:
log.debug("skipping byte-compilation of %s to %s", file,
cfile_base)
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
if not self.initialized:
self.initialize()
(objects, output_dir) = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
(libraries, library_dirs, runtime_library_dirs) = fixed_args
if runtime_library_dirs:
self.warn ("I don't know what to do with 'runtime_library_dirs': "
+ str (runtime_library_dirs))
lib_opts = gen_lib_options(self,
library_dirs, runtime_library_dirs,
libraries)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
if debug:
ldflags = self.ldflags_shared_debug[1:]
else:
ldflags = self.ldflags_shared[1:]
else:
if debug:
ldflags = self.ldflags_shared_debug
else:
ldflags = self.ldflags_shared
export_opts = []
for sym in (export_symbols or []):
export_opts.append("/EXPORT:" + sym)
ld_args = (ldflags + lib_opts + export_opts +
objects + ['/OUT:' + output_filename])
# The MSVC linker generates .lib and .exp files, which cannot be
# suppressed by any linker switches. The .lib files may even be
# needed! Make sure they are generated in the temporary build
# directory. Since they have different names for debug and release
# builds, they can go into the same directory.
build_temp = os.path.dirname(objects[0])
if export_symbols is not None:
(dll_name, dll_ext) = os.path.splitext(
os.path.basename(output_filename))
implib_file = os.path.join(
build_temp,
self.library_filename(dll_name))
ld_args.append ('/IMPLIB:' + implib_file)
self.manifest_setup_ldargs(output_filename, build_temp, ld_args)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
# embed the manifest
# XXX - this is somewhat fragile - if mt.exe fails, distutils
# will still consider the DLL up-to-date, but it will not have a
# manifest. Maybe we should link to a temp file? OTOH, that
# implies a build environment error that shouldn't go undetected.
mfinfo = self.manifest_get_embed_info(target_desc, ld_args)
if mfinfo is not None:
mffilename, mfid = mfinfo
out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
try:
self.spawn(['mt.exe', '-nologo', '-manifest',
mffilename, out_arg])
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug("skipping %s (up-to-date)", output_filename)
def build_dlib():
"""use cmake to build and install the extension
"""
if cmake_path is None:
cmake_install_url = "https://cmake.org/install/"
message = ("You can install cmake using the instructions at " +
cmake_install_url)
msg_pkgmanager = ("You can install cmake on {0} using "
"`sudo {1} install cmake`.")
if sys.platform == "darwin":
pkgmanagers = ('brew', 'port')
for manager in pkgmanagers:
if find_executable(manager) is not None:
message = msg_pkgmanager.format('OSX', manager)
break
elif sys.platform.startswith('linux'):
try:
import distro
except ImportError as err:
import pip
pip_exit = pip.main(['install', '-q', 'distro'])
if pip_exit > 0:
log.debug("Unable to install `distro` to identify "
"the recommended command. Falling back "
"to default error message.")
distro = err
else:
import distro
if not isinstance(distro, ImportError):
distname = distro.id()
if distname in ('debian', 'ubuntu'):
message = msg_pkgmanager.format(
distname.title(), 'apt-get')
elif distname in ('fedora', 'centos', 'redhat'):
pkgmanagers = ("dnf", "yum")
for manager in pkgmanagers:
if find_executable(manager) is not None:
message = msg_pkgmanager.format(
distname.title(), manager)
break
raise DistutilsSetupError(
"Cannot find cmake, ensure it is installed and in the path.\n"
+ message + "\n"
"You can also specify its path with --cmake parameter.")
platform_arch = platform.architecture()[0]
log.info("Detected Python architecture: %s" % platform_arch)
# make sure build artifacts are generated for the version of Python currently running
cmake_extra_arch = []
inc_dir = get_python_inc()
lib_dir = get_config_var('LIBDIR')
if (inc_dir != None):
cmake_extra_arch += ['-DPYTHON_INCLUDE_DIR=' + inc_dir]
if (lib_dir != None):
cmake_extra_arch += ['-DCMAKE_LIBRARY_PATH=' + lib_dir]
if sys.version_info >= (3, 0):
cmake_extra_arch += ['-DPYTHON3=yes']
log.info("Detected platform: %s" % sys.platform)
if sys.platform == "darwin":
# build on OS X
# by default, cmake will choose the system python lib in /usr/lib
# this checks the sysconfig and will correctly pick up a brewed python lib
# e.g. in /usr/local/Cellar
py_ver = get_python_version()
# check: in some virtual environments the libpython has the form "libpython_#m.dylib
py_lib = os.path.join(get_config_var('LIBDIR'),
'libpython' + py_ver + '.dylib')
if not os.path.isfile(py_lib):
py_lib = os.path.join(get_config_var('LIBDIR'),
'libpython' + py_ver + 'm.dylib')
cmake_extra_arch += ['-DPYTHON_LIBRARY={lib}'.format(lib=py_lib)]
if sys.platform == "win32":
if platform_arch == '64bit' and not generator_set:
cmake_extra_arch += get_msvc_win64_generator()
# this imitates cmake in path resolution
py_ver = get_python_version()
for ext in [
py_ver.replace(".", "") + '.lib', py_ver + 'mu.lib',
py_ver + 'm.lib', py_ver + 'u.lib'
]:
py_lib = os.path.abspath(
os.path.join(inc_dir, '../libs/', 'python' + ext))
if os.path.exists(py_lib):
cmake_extra_arch += [
'-DPYTHON_LIBRARY={lib}'.format(lib=py_lib)
]
break
build_dir = os.path.join(script_dir, "./tools/python/build")
if os.path.exists(build_dir):
log.info('Removing build directory %s' % build_dir)
rmtree(build_dir)
try:
os.makedirs(build_dir)
except OSError:
pass
# cd build
os.chdir(build_dir)
log.info('Configuring cmake ...')
cmake_cmd = [
cmake_path,
"..",
] + cmake_extra + cmake_extra_arch
if run_process(cmake_cmd):
raise DistutilsSetupError("cmake configuration failed!")
log.info('Build using cmake ...')
cmake_cmd = [
cmake_path,
"--build",
".",
"--config",
cmake_config,
"--target",
"install",
]
if run_process(cmake_cmd):
raise DistutilsSetupError("cmake build failed!")
# cd back where setup awaits
os.chdir(script_dir)
def build_executable(self, exe):
ListType, TupleType = type([]), type(())
from distutils.dep_util import newer_group
sources = exe.sources
if sources is None or type(sources) not in (ListType, TupleType):
raise DistutilsSetupError(
("in 'executables' option (executable '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % exe.name)
sources = list(sources)
exe_filename = os.path.join(self.build_exe, exe.name)
depends = sources + exe.depends
if not (self.force or newer_group(depends, exe_filename, 'newer')):
log.debug("skipping '%s' executable (up-to-date)", exe.name)
return
else:
log.info("building '%s' executable", exe.name)
# Next, compile the source code to object files.
# XXX not honouring 'define_macros' or 'undef_macros' -- the
# CCompiler API needs to change to accommodate this, and I
# want to do one thing at a time!
macros = exe.define_macros[:]
for undef in exe.undef_macros:
macros.append((undef, ))
# Two possible sources for extra compiler arguments:
# - 'extra_compile_args' in Extension object
# - CFLAGS environment variable (not particularly
# elegant, but people seem to expect it and I
# guess it's useful)
# The environment variable should take precedence, and
# any sensible compiler will give precedence to later
# command line args. Hence we combine them in order:
extra_args = exe.extra_compile_args or []
extra_args = extra_args[:]
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=exe.include_dirs,
debug=self.debug,
extra_postargs=extra_args,
depends=exe.depends)
# XXX -- this is a Vile HACK!
#
# The setup.py script for Python on Unix needs to be able to
# get this list so it can perform all the clean up needed to
# avoid keeping object files around when cleaning out a failed
# build of an extension module. Since Distutils does not
# track dependencies, we have to get rid of intermediates to
# ensure all the intermediates will be properly re-built.
#
self._built_objects = objects[:]
# XXX -- this is a Vile HACK!
#
# Remove msvcrXX.dll when building executables with MinGW
#
if self.compiler.compiler_type == 'mingw32':
try:
del self.compiler.dll_libraries[:]
except:
pass
# Now link the object files together into a "shared object" --
# of course, first we have to figure out all the other things
# that go into the mix.
if exe.extra_objects:
objects.extend(exe.extra_objects)
extra_args = exe.extra_link_args or []
extra_args = extra_args[:]
# Get special linker flags for building a executable with
# bundled Python library, also fix location of needed
# python.exp file on AIX
ldshflag = sysconfig.get_config_var('LINKFORSHARED') or ''
if sys.platform.startswith('aix'):
python_lib = sysconfig.get_python_lib(standard_lib=1)
python_exp = os.path.join(python_lib, 'config', 'python.exp')
ldshflag = ldshflag.replace('Modules/python.exp', python_exp)
extra_args.extend(ldshflag.split())
# Detect target language, if not provided
language = exe.language or self.compiler.detect_language(sources)
self.compiler.link_executable(
objects,
exe_filename,
output_dir=None,
libraries=self.get_libraries(exe),
library_dirs=exe.library_dirs,
runtime_library_dirs=exe.runtime_library_dirs,
extra_preargs=None,
extra_postargs=extra_args,
debug=self.debug,
target_lang=language)
for base in HKEYS:
if read_values(base, r"%s\6.0" % self.__root) is not None:
self.warn("It seems you have Visual Studio 6 installed, "
"but the expected registry settings are not present.\n"
"You must at least run the Visual Studio GUI once "
"so that these entries are created.")
break
return []
def set_path_env_var(self, name):
"""Set environment variable 'name' to an MSVC path type value.
This is equivalent to a SET command prior to execution of spawned
commands.
"""
if name == "lib":
p = self.get_msvc_paths("library")
else:
p = self.get_msvc_paths(name)
if p:
os.environ[name] = ';'.join(p)
if get_build_version() >= 8.0:
log.debug("Importing new compiler from distutils.msvc9compiler")
OldMSVCCompiler = MSVCCompiler
from distutils.msvc9compiler import MSVCCompiler
# get_build_architecture not really relevant now we support cross-compile
from distutils.msvc9compiler import MacroExpander
if not self.initialized: self.initialize()
(objects, output_dir) = self._fix_object_args(objects, output_dir)
output_filename = \
self.library_filename (output_libname, output_dir=output_dir)
if self._need_link(objects, output_filename):
lib_args = objects + ['/OUT:' + output_filename]
if debug:
pass # XXX what goes here?
try:
self.spawn([self.lib] + lib_args)
except DistutilsExecError, msg:
raise LibError, msg
else:
log.debug("skipping %s (up-to-date)", output_filename)
# create_static_lib ()
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
def find_vcvarsall(version):
"""Find the vcvarsall.bat file
At first it tries to find the productdir of VS 2008 in the registry. If
that fails it falls back to the VS90COMNTOOLS env var.
"""
vsbase = VS_BASE % version
try:
productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
"productdir")
except KeyError:
log.debug("Unable to find productdir in registry")
productdir = None
# trying Express edition
if productdir is None:
vsbase = VSEXPRESS_BASE % version
try:
productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
"productdir")
except KeyError:
productdir = None
log.debug("Unable to find productdir in registry")
if not productdir or not os.path.isdir(productdir):
toolskey = "VS%0.f0COMNTOOLS" % version
toolsdir = os.environ.get(toolskey, None)
if toolsdir and os.path.isdir(toolsdir):
productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
productdir = os.path.abspath(productdir)
if not os.path.isdir(productdir):
log.debug("%s is not a valid directory" % productdir)
return None
else:
log.debug("Env var %s is not set or invalid" % toolskey)
if not productdir:
log.debug("No productdir found")
return None
vcvarsall = os.path.join(productdir, "vcvarsall.bat")
if os.path.isfile(vcvarsall):
return vcvarsall
log.debug("Unable to find vcvarsall.bat")
return None
def copy_file(src,
dst,
preserve_mode=1,
preserve_times=1,
update=0,
link=None,
verbose=0,
dry_run=0):
"""Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src' is
copied there with the same name; otherwise, it must be a filename. (If
the file exists, it will be ruthlessly clobbered.) If 'preserve_mode'
is true (the default), the file's mode (type and permission bits, or
whatever is analogous on the current platform) is copied. If
'preserve_times' is true (the default), the last-modified and
last-access times are copied as well. If 'update' is true, 'src' will
only be copied if 'dst' does not exist, or if 'dst' does exist but is
older than 'src'.
'link' allows you to make hard links (os.link) or symbolic links
(os.symlink) instead of copying: set it to "hard" or "sym"; if it is
None (the default), files are copied. Don't set 'link' on systems that
don't support it: 'copy_file()' doesn't check if hard or symbolic
linking is available.
Under Mac OS, uses the native file copy function in macostools; on
other systems, uses '_copy_file_contents()' to copy file contents.
Return a tuple (dest_name, copied): 'dest_name' is the actual name of
the output file, and 'copied' is true if the file was copied (or would
have been copied, if 'dry_run' true).
"""
# XXX if the destination file already exists, we clobber it if
# copying, but blow up if linking. Hmmm. And I don't know what
# macostools.copyfile() does. Should definitely be consistent, and
# should probably blow up if destination exists and we would be
# changing it (ie. it's not already a hard/soft link to src OR
# (not update) and (src newer than dst).
from distutils.dep_util import newer
from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE
if not os.path.isfile(src):
raise DistutilsFileError, \
"can't copy '%s': doesn't exist or not a regular file" % src
if os.path.isdir(dst):
dir = dst
dst = os.path.join(dst, os.path.basename(src))
else:
dir = os.path.dirname(dst)
if update and not newer(src, dst):
log.debug("not copying %s (output up-to-date)", src)
return dst, 0
try:
action = _copy_action[link]
except KeyError:
raise ValueError, \
"invalid value '%s' for 'link' argument" % link
if os.path.basename(dst) == os.path.basename(src):
log.info("%s %s -> %s", action, src, dir)
else:
log.info("%s %s -> %s", action, src, dst)
if dry_run:
return (dst, 1)
# On Mac OS, use the native file copy routine
if os.name == 'mac':
import macostools
try:
macostools.copy(src, dst, 0, preserve_times)
except os.error, exc:
raise DistutilsFileError, \
"could not copy '%s' to '%s': %s" % (src, dst, exc[-1])
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = self._fix_lib_args(
libraries, library_dirs, runtime_library_dirs)
if runtime_library_dirs:
log.warn("I don't know what to do with 'runtime_library_dirs': %s",
str(runtime_library_dirs))
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
startup_obj = 'c0w32'
if debug:
ld_args = self.ldflags_exe_debug[:]
else:
ld_args = self.ldflags_exe[:]
else:
startup_obj = 'c0d32'
if debug:
ld_args = self.ldflags_shared_debug[:]
else:
ld_args = self.ldflags_shared[:]
if export_symbols is None:
def_file = ''
else:
head, tail = os.path.split(output_filename)
modname, ext = os.path.splitext(tail)
temp_dir = os.path.dirname(objects[0])
def_file = os.path.join(temp_dir, '%s.def' % modname)
contents = ['EXPORTS']
for sym in export_symbols or []:
contents.append(' %s=_%s' % (sym, sym))
self.execute(write_file, (def_file, contents),
'writing %s' % def_file)
objects2 = map(os.path.normpath, objects)
objects = [startup_obj]
resources = []
for file in objects2:
base, ext = os.path.splitext(os.path.normcase(file))
if ext == '.res':
resources.append(file)
objects.append(file)
for l in library_dirs:
ld_args.append('/L%s' % os.path.normpath(l))
ld_args.append('/L.')
ld_args.extend(objects)
ld_args.extend([',', output_filename])
ld_args.append(',,')
for lib in libraries:
libfile = self.find_library_file(library_dirs, lib, debug)
if libfile is None:
ld_args.append(lib)
ld_args.append(libfile)
ld_args.append('import32')
ld_args.append('cw32mt')
ld_args.extend([',', def_file])
ld_args.append(',')
ld_args.extend(resources)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except DistutilsExecError as msg:
raise LinkError, msg
else:
log.debug('skipping %s (up-to-date)', output_filename)
return
def run_prospector(base_dir, clear_ignore_patterns=False):
"""Run prospector and apply white/blacklists to the results"""
orig_expand_default = optparse.HelpFormatter.expand_default
sys.argv = ['fakename']
sys.argv.extend(PROSPECTOR_OPTIONS)
if PROSPECTOR_USE_LIBS:
sys.argv.extend(["--uses", ",".join(PROSPECTOR_USE_LIBS)])
# add/set REPO_BASE_DIR as positional path
sys.argv.append(base_dir)
log.debug("prospector commandline %s" % sys.argv)
config = ProspectorConfig()
# prospector will sometimes wrongly autodetect django
config.libraries = []
prospector = Prospector(config)
# Set defaults for prospector:
config.profile.doc_warnings = False
config.profile.test_warnings = False
config.profile.autodetect = True
config.profile.member_warnings = False
if clear_ignore_patterns:
config.profile.ignore_patterns = [r'.^']
config.ignores = []
else:
append_pattern = r'(^|/)\..+'
config.profile.ignore_patterns.append(append_pattern)
config.ignores.append(re.compile(append_pattern))
# Enable pylint Python3 compatibility tests:
config.profile.pylint['options']['enable'] = 'python3'
log.debug("prospector argv = %s" % sys.argv)
log.debug("prospector profile from config = %s" % vars(config.profile))
prospector.execute()
log.debug("prospector profile form prospector = %s" % vars(prospector.config.profile))
blacklist = map(re.compile, PROSPECTOR_BLACKLIST)
whitelist = map(re.compile, PROSPECTOR_WHITELIST)
failures = []
for msg in prospector.get_messages():
# example msg.as_dict():
# {'source': 'pylint', 'message': 'Missing function docstring', 'code': 'missing-docstring',
# 'location': {'function': 'TestHeaders.test_check_header.lgpl', 'path': u'headers.py',
# 'line': 122, 'character': 8, 'module': 'headers'}}
log.debug("prospector message %s" % msg.as_dict())
if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in blacklist]):
continue
if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in whitelist]):
failures.append(msg.as_dict())
# There is some ugly monkeypatch code in pylint
# (or logilab if no recent enough pylint is installed)
# Make sure the original is restored
# (before any errors are reported; no need to put this in setUp/tearDown)
optparse.HelpFormatter.expand_default = orig_expand_default
return failures
def run(self):
if (sys.platform != "win32"
and (self.distribution.has_ext_modules()
or self.distribution.has_c_libraries())):
raise DistutilsPlatformError \
("distribution contains extensions and/or C libraries; "
"must be compiled on a Windows 32 platform")
if not self.skip_build:
self.run_command('build')
install = self.reinitialize_command('install', reinit_subcommands=1)
install.root = self.bdist_dir
install.skip_build = self.skip_build
install.warn_dir = 0
install_lib = self.reinitialize_command('install_lib')
# we do not want to include pyc or pyo files
install_lib.compile = 0
install_lib.optimize = 0
if self.distribution.has_ext_modules():
# If we are building an installer for a Python version other
# than the one we are currently running, then we need to ensure
# our build_lib reflects the other Python version rather than ours.
# Note that for target_version!=sys.version, we must have skipped the
# build step, so there is no issue with enforcing the build of this
# version.
target_version = self.target_version
if not target_version:
assert self.skip_build, "Should have already checked this"
target_version = sys.version[0:3]
plat_specifier = ".%s-%s" % (get_platform(), target_version)
build = self.get_finalized_command('build')
build.build_lib = os.path.join(build.build_base,
'lib' + plat_specifier)
# Use a custom scheme for the zip-file, because we have to decide
# at installation time which scheme to use.
for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
value = string.upper(key)
if key == 'headers':
value = value + '/Include/$dist_name'
setattr(install, 'install_' + key, value)
log.info("installing to %s", self.bdist_dir)
install.ensure_finalized()
# avoid warning of 'install_lib' about installing
# into a directory not in sys.path
sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))
install.run()
del sys.path[0]
# And make an archive relative to the root of the
# pseudo-installation tree.
from tempfile import mktemp
archive_basename = mktemp()
fullname = self.distribution.get_fullname()
arcname = self.make_archive(archive_basename,
"zip",
root_dir=self.bdist_dir)
# create an exe containing the zip-file
self.create_exe(arcname, fullname, self.bitmap)
if self.distribution.has_ext_modules():
pyversion = get_python_version()
else:
pyversion = 'any'
self.distribution.dist_files.append(
('bdist_wininst', pyversion,
self.get_installer_filename(fullname)))
# remove the zip-file again
log.debug("removing temporary file '%s'", arcname)
os.remove(arcname)
if not self.keep_temp:
remove_tree(self.bdist_dir, dry_run=self.dry_run)
def _run_yarn_command(self, cmd, env=None):
log.debug(u'yarn path: ({0}))'.format(YARN_PATH))
self._run_command(
[YARN_PATH] + cmd, env=env
)
def link (self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
# First fixup.
(objects, output_dir) = self._fix_object_args (objects, output_dir)
(libraries, library_dirs, runtime_library_dirs) = \
self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)
# First examine a couple of options for things that aren't implemented yet
if not target_desc in (self.SHARED_LIBRARY, self.SHARED_OBJECT):
raise DistutilsPlatformError, 'Can only make SHARED_LIBRARY or SHARED_OBJECT targets on the Mac'
if runtime_library_dirs:
raise DistutilsPlatformError, 'Runtime library dirs not implemented yet'
if extra_preargs or extra_postargs:
raise DistutilsPlatformError, 'Runtime library dirs not implemented yet'
if len(export_symbols) != 1:
raise DistutilsPlatformError, 'Need exactly one export symbol'
# Next there are various things for which we need absolute pathnames.
# This is because we (usually) create the project in a subdirectory of
# where we are now, and keeping the paths relative is too much work right
# now.
sources = map(self._filename_to_abs, self.__sources)
include_dirs = map(self._filename_to_abs, self.__include_dirs)
if objects:
objects = map(self._filename_to_abs, objects)
else:
objects = []
if build_temp:
build_temp = self._filename_to_abs(build_temp)
else:
build_temp = os.curdir()
if output_dir:
output_filename = os.path.join(output_dir, output_filename)
# The output filename needs special handling: splitting it into dir and
# filename part. Actually I'm not sure this is really needed, but it
# can't hurt.
output_filename = self._filename_to_abs(output_filename)
output_dir, output_filename = os.path.split(output_filename)
# Now we need the short names of a couple of things for putting them
# into the project.
if output_filename[-8:] == '.ppc.slb':
basename = output_filename[:-8]
elif output_filename[-11:] == '.carbon.slb':
basename = output_filename[:-11]
else:
basename = os.path.strip(output_filename)[0]
projectname = basename + '.mcp'
targetname = basename
xmlname = basename + '.xml'
exportname = basename + '.mcp.exp'
prefixname = 'mwerks_%s_config.h'%basename
# Create the directories we need
distutils.dir_util.mkpath(build_temp, dry_run=self.dry_run)
distutils.dir_util.mkpath(output_dir, dry_run=self.dry_run)
# And on to filling in the parameters for the project builder
settings = {}
settings['mac_exportname'] = exportname
settings['mac_outputdir'] = output_dir
settings['mac_dllname'] = output_filename
settings['mac_targetname'] = targetname
settings['sysprefix'] = sys.prefix
settings['mac_sysprefixtype'] = 'Absolute'
sourcefilenames = []
sourcefiledirs = []
for filename in sources + objects:
dirname, filename = os.path.split(filename)
sourcefilenames.append(filename)
if not dirname in sourcefiledirs:
sourcefiledirs.append(dirname)
settings['sources'] = sourcefilenames
settings['libraries'] = libraries
settings['extrasearchdirs'] = sourcefiledirs + include_dirs + library_dirs
if self.dry_run:
print 'CALLING LINKER IN', os.getcwd()
for key, value in settings.items():
print '%20.20s %s'%(key, value)
return
# Build the export file
exportfilename = os.path.join(build_temp, exportname)
log.debug("\tCreate export file %s", exportfilename)
fp = open(exportfilename, 'w')
fp.write('%s\n'%export_symbols[0])
fp.close()
# Generate the prefix file, if needed, and put it in the settings
if self.__macros:
prefixfilename = os.path.join(os.getcwd(), os.path.join(build_temp, prefixname))
fp = open(prefixfilename, 'w')
fp.write('#include "mwerks_shcarbon_config.h"\n')
for name, value in self.__macros:
if value is None:
fp.write('#define %s\n'%name)
else:
fp.write('#define %s %s\n'%(name, value))
fp.close()
settings['prefixname'] = prefixname
# Build the XML file. We need the full pathname (only lateron, really)
# because we pass this pathname to CodeWarrior in an AppleEvent, and CW
# doesn't have a clue about our working directory.
xmlfilename = os.path.join(os.getcwd(), os.path.join(build_temp, xmlname))
log.debug("\tCreate XML file %s", xmlfilename)
xmlbuilder = mkcwproject.cwxmlgen.ProjectBuilder(settings)
xmlbuilder.generate()
xmldata = settings['tmp_projectxmldata']
fp = open(xmlfilename, 'w')
fp.write(xmldata)
fp.close()
# Generate the project. Again a full pathname.
projectfilename = os.path.join(os.getcwd(), os.path.join(build_temp, projectname))
log.debug('\tCreate project file %s', projectfilename)
mkcwproject.makeproject(xmlfilename, projectfilename)
# And build it
log.debug('\tBuild project')
mkcwproject.buildproject(projectfilename)
def run(self):
if has_system_lib():
log.info('Using system library')
return
build_temp = os.path.abspath(self.build_temp)
try:
os.makedirs(build_temp)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.exists(absolute('libsecp256k1')):
# library needs to be downloaded
self.get_source_files()
if not os.path.exists(absolute('libsecp256k1/configure')):
# configure script hasn't been generated yet
autogen = absolute('libsecp256k1/autogen.sh')
os.chmod(absolute(autogen), 0o755)
subprocess.check_call([autogen], cwd=absolute('libsecp256k1'))
for filename in [
'libsecp256k1/configure',
'libsecp256k1/build-aux/compile',
'libsecp256k1/build-aux/config.guess',
'libsecp256k1/build-aux/config.sub',
'libsecp256k1/build-aux/depcomp',
'libsecp256k1/build-aux/install-sh',
'libsecp256k1/build-aux/missing',
'libsecp256k1/build-aux/test-driver',
]:
try:
os.chmod(absolute(filename), 0o755)
except OSError as e:
# some of these files might not exist depending on autoconf version
if e.errno != errno.ENOENT:
# If the error isn't 'No such file or directory' something
# else is wrong and we want to know about it
raise
cmd = [
absolute('libsecp256k1/configure'),
'--disable-shared',
'--enable-static',
'--disable-dependency-tracking',
'--with-pic',
'--enable-module-recovery',
'--disable-jni',
'--prefix',
os.path.abspath(self.build_clib),
'--enable-experimental',
'--enable-module-ecdh',
'--enable-benchmark=no',
# '--enable-endomorphism',
]
log.debug('Running configure: {}'.format(' '.join(cmd)))
subprocess.check_call(cmd, cwd=build_temp)
subprocess.check_call([MAKE], cwd=build_temp)
subprocess.check_call([MAKE, 'install'], cwd=build_temp)
self.build_flags['include_dirs'].extend(
build_flags('libsecp256k1', 'I', build_temp))
self.build_flags['library_dirs'].extend(
build_flags('libsecp256k1', 'L', build_temp))
if not has_system_lib():
self.build_flags['define'].append(('CFFI_ENABLE_RECOVERY', None))
else:
pass
def finalize_options(self):
# This requires some explanation. Basically what we want to do
# here is to control if we want to build in-place or into the
# build-lib folder. Traditionally this is set by the `inplace`
# command line flag for build_ext. However as we are a subcommand
# we need to grab this information from elsewhere.
#
# An in-place build puts the files generated into the source
# folder, a regular build puts the files into the build-lib
# folder.
#
# The following situations we need to cover:
#
# command default in-place
# setup.py build_js 0
# setup.py build_ext value of in-place for build_ext
# setup.py build_ext --inplace 1
# pip install --editable . 1
# setup.py install 0
# setup.py sdist 0
# setup.py bdist_wheel 0
#
# The way this is achieved is that build_js is invoked by two
# subcommands: bdist_ext (which is in our case always executed
# due to a custom distribution) or sdist.
#
# Note: at one point install was an in-place build but it's not
# quite sure why. In case a version of install breaks again:
# installations via pip from git URLs definitely require the
# in-place flag to be disabled. So we might need to detect
# that separately.
#
# To find the default value of the inplace flag we inspect the
# sdist and build_ext commands.
sdist = self.distribution.get_command_obj('sdist')
build_ext = self.get_finalized_command('build_ext')
# If we are not decided on in-place we are inplace if either
# build_ext is inplace or we are invoked through the install
# command (easiest check is to see if it's finalized).
if self.inplace is None:
self.inplace = (build_ext.inplace or sdist.finalized) and 1 or 0
# If we're coming from sdist, clear the hell out of the dist
# folder first.
if sdist.finalized:
for path in self.get_dist_paths():
try:
shutil.rmtree(path)
except (OSError, IOError):
pass
# In place means build_lib is src. We also log this.
if self.inplace:
log.debug('in-place js building enabled')
self.build_lib = 'src'
# Otherwise we fetch build_lib from the build command.
else:
self.set_undefined_options('build', ('build_lib', 'build_lib'))
log.debug('regular js build: build path is %s' % self.build_lib)
if self.work_path is None:
self.work_path = self.get_root_path()
def link(self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
if not self.initialized:
self.initialize()
objects, output_dir = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
libraries, library_dirs, runtime_library_dirs = fixed_args
if runtime_library_dirs:
self.warn("I don't know what to do with 'runtime_library_dirs': " +
str(runtime_library_dirs))
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
if debug:
ldflags = self.ldflags_shared_debug[1:]
else:
ldflags = self.ldflags_shared[1:]
elif debug:
ldflags = self.ldflags_shared_debug
else:
ldflags = self.ldflags_shared
export_opts = []
for sym in export_symbols or []:
export_opts.append('/EXPORT:' + sym)
ld_args = ldflags + lib_opts + export_opts + objects + [
'/OUT:' + output_filename
]
build_temp = os.path.dirname(objects[0])
if export_symbols is not None:
dll_name, dll_ext = os.path.splitext(
os.path.basename(output_filename))
implib_file = os.path.join(build_temp,
self.library_filename(dll_name))
ld_args.append('/IMPLIB:' + implib_file)
self.manifest_setup_ldargs(output_filename, build_temp, ld_args)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except DistutilsExecError as msg:
raise LinkError(msg)
mfinfo = self.manifest_get_embed_info(target_desc, ld_args)
if mfinfo is not None:
mffilename, mfid = mfinfo
out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
try:
self.spawn([
'mt.exe', '-nologo', '-manifest', mffilename, out_arg
])
except DistutilsExecError as msg:
raise LinkError(msg)
else:
log.debug('skipping %s (up-to-date)', output_filename)
return
def copy_file(src,
dst,
preserve_mode=1,
preserve_times=1,
update=0,
link=None,
verbose=1,
dry_run=0):
"""Copy a file 'src' to 'dst'.
If 'dst' is a directory, then 'src' is copied there with the same name;
otherwise, it must be a filename. (If the file exists, it will be
ruthlessly clobbered.) If 'preserve_mode' is true (the default),
the file's mode (type and permission bits, or whatever is analogous on
the current platform) is copied. If 'preserve_times' is true (the
default), the last-modified and last-access times are copied as well.
If 'update' is true, 'src' will only be copied if 'dst' does not exist,
or if 'dst' does exist but is older than 'src'.
'link' allows you to make hard links (os.link) or symbolic links
(os.symlink) instead of copying: set it to "hard" or "sym"; if it is
None (the default), files are copied. Don't set 'link' on systems that
don't support it: 'copy_file()' doesn't check if hard or symbolic
linking is available.
Under Mac OS, uses the native file copy function in macostools; on
other systems, uses '_copy_file_contents()' to copy file contents.
Return a tuple (dest_name, copied): 'dest_name' is the actual name of
the output file, and 'copied' is true if the file was copied (or would
have been copied, if 'dry_run' true).
"""
# XXX if the destination file already exists, we clobber it if
# copying, but blow up if linking. Hmmm. And I don't know what
# macostools.copyfile() does. Should definitely be consistent, and
# should probably blow up if destination exists and we would be
# changing it (ie. it's not already a hard/soft link to src OR
# (not update) and (src newer than dst).
from distutils.dep_util import newer
from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE
if not os.path.isfile(src):
raise DistutilsFileError(
"can't copy '%s': doesn't exist or not a regular file" % src)
if os.path.isdir(dst):
dir = dst
dst = os.path.join(dst, os.path.basename(src))
else:
dir = os.path.dirname(dst)
if update and not newer(src, dst):
if verbose >= 1:
log.debug("not copying %s (output up-to-date)", src)
return dst, 0
try:
action = _copy_action[link]
except KeyError:
raise ValueError("invalid value '%s' for 'link' argument" % link)
if verbose >= 1:
if os.path.basename(dst) == os.path.basename(src):
log.info("%s %s -> %s", action, src, dir)
else:
log.info("%s %s -> %s", action, src, dst)
if dry_run:
return (dst, 1)
# If linking (hard or symbolic), use the appropriate system call
# (Unix only, of course, but that's the caller's responsibility)
if link == 'hard':
if not (os.path.exists(dst) and os.path.samefile(src, dst)):
os.link(src, dst)
elif link == 'sym':
if not (os.path.exists(dst) and os.path.samefile(src, dst)):
os.symlink(src, dst)
# Otherwise (non-Mac, not linking), copy the file contents and
# (optionally) copy the times and mode.
else:
_copy_file_contents(src, dst)
if preserve_mode or preserve_times:
st = os.stat(src)
# According to David Ascher <*****@*****.**>, utime() should be done
# before chmod() (at least under NT).
if preserve_times:
os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
if preserve_mode and hasattr(os, 'chmod'):
os.chmod(dst, S_IMODE(st[ST_MODE]))
return (dst, 1)
from sage_setup.optional_extension import is_package_installed_and_updated
optional_packages_with_extensions = ['mcqd', 'bliss', 'tdlib', 'primecount',
'coxeter3', 'fes', 'sirocco', 'meataxe']
distributions += ['sage-{}'.format(pkg)
for pkg in optional_packages_with_extensions
if is_package_installed_and_updated(pkg)]
log.warn('distributions = {0}'.format(distributions))
from sage_setup.find import find_python_sources
python_packages, python_modules, cython_modules = find_python_sources(
SAGE_SRC, ['sage', 'sage_setup'], distributions=distributions)
log.debug('python_packages = {0}'.format(python_packages))
print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
from sage_setup.command.sage_install import sage_install
#########################################################
### Distutils
#########################################################
code = setup(name = 'sage',
version = SAGE_VERSION,
description = 'Sage: Open Source Mathematics Software',
license = 'GNU Public License (GPL)',
author = 'William Stein et al.',
def build_extension(self, ext):
"""Modified version of build_extension method from distutils.
Can handle compiler args for different files"""
sources = ext.sources
if sources is None or not isinstance(sources, (list, tuple)):
raise DistutilsSetupError("in 'ext_modules' option (extension '%s'), "
"'sources' must be present and must be "
"a list of source filenames" % ext.name)
sources = list(sources)
ext_path = self.get_ext_fullpath(ext.name)
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_path, 'newer')):
log.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
log.info("building '%s' extension", ext.name)
sources = self.swig_sources(sources, ext)
extra_args = ext.extra_compile_args or []
extra_c_args = getattr(ext, "extra_compile_c_args", [])
extra_cpp_args = getattr(ext, "extra_compile_cpp_args", [])
extra_objc_args = getattr(ext, "extra_compile_objc_args", [])
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef, ))
c_sources, cpp_sources, objc_sources, other_sources = filter_sources(
sources)
def _compile(src, args):
return self.compiler.compile(src,
output_dir=self.build_temp,
macros=macros,
include_dirs=ext.include_dirs,
debug=self.debug,
extra_postargs=extra_args + args,
depends=ext.depends)
objects = []
objects += _compile(c_sources, extra_c_args)
objects += _compile(cpp_sources, extra_cpp_args)
objects += _compile(objc_sources, extra_objc_args)
objects += _compile(other_sources, [])
self._built_objects = objects[:]
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
language = ext.language or self.compiler.detect_language(sources)
self.compiler.link_shared_object(
objects,
ext_path,
libraries=self.get_libraries(ext),
library_dirs=ext.library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=language)
