def _find_stale_files(site_packages, python_packages, python_modules,
ext_modules):
"""
Find stale files
This method lists all files installed and then subtracts the ones
which are intentionally being installed.
EXAMPLES:
It is crucial that only truly stale files are being found, of
course. We check that when the doctest is being run, that is,
after installation, there are no stale files::
sage: from sage.env import SAGE_SRC, SAGE_LIB
sage: from sage_setup.find import find_python_sources
sage: python_packages, python_modules = find_python_sources(
....: SAGE_SRC, ['sage', 'sage_setup'])
sage: from sage_setup.clean import _find_stale_files
TODO: move ``module_list.py`` into ``sage_setup`` and also check
extension modules::
sage: stale_iter = _find_stale_files(SAGE_LIB, python_packages, python_modules, [])
sage: from sage.misc.sageinspect import loadable_module_extension
sage: for f in stale_iter:
....: if f.endswith(loadable_module_extension()): continue
....: print('Found stale file: ' + f)
"""
PYMOD_EXTS = (os.path.extsep + 'py', os.path.extsep + 'pyc')
from sage.misc.sageinspect import loadable_module_extension
CEXTMOD_EXTS = (loadable_module_extension(), )
INIT_FILES = map(lambda x: '__init__' + x, PYMOD_EXTS)
module_files = installed_files_by_module(site_packages,
['sage', 'sage_setup'])
for mod in python_packages:
try:
files = module_files[mod]
except KeyError:
# the source module "mod" has not been previously installed, fine.
continue
_remove(files, mod, INIT_FILES)
for mod in python_modules:
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, PYMOD_EXTS)
for ext in ext_modules:
mod = ext.name
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, CEXTMOD_EXTS)
for files in module_files.values():
for f in files:
yield f
def _find_stale_files(site_packages, python_packages, python_modules, ext_modules):
"""
Find stale files
This method lists all files installed and then subtracts the ones
which are intentionally being installed.
EXAMPLES:
It is crucial that only truly stale files are being found, of
course. We check that when the doctest is being run, that is,
after installation, there are no stale files::
sage: from sage.env import SAGE_SRC, SAGE_LIB
sage: from sage_setup.find import find_python_sources
sage: python_packages, python_modules = find_python_sources(
....: SAGE_SRC, ['sage', 'sage_setup'])
sage: from sage_setup.clean import _find_stale_files
TODO: move ``module_list.py`` into ``sage_setup`` and also check
extension modules::
sage: stale_iter = _find_stale_files(SAGE_LIB, python_packages, python_modules, [])
sage: from sage.misc.sageinspect import loadable_module_extension
sage: for f in stale_iter:
....: if f.endswith(loadable_module_extension()): continue
....: print('Found stale file: ' + f)
"""
PYMOD_EXTS = (os.path.extsep + 'py', os.path.extsep + 'pyc')
from sage.misc.sageinspect import loadable_module_extension
CEXTMOD_EXTS = (loadable_module_extension(),)
INIT_FILES= map(lambda x: '__init__' + x, PYMOD_EXTS)
module_files = installed_files_by_module(site_packages, ['sage', 'sage_setup'])
for mod in python_packages:
try:
files = module_files[mod]
except KeyError:
# the source module "mod" has not been previously installed, fine.
continue
_remove(files, mod, INIT_FILES)
for mod in python_modules:
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, PYMOD_EXTS)
for ext in ext_modules:
mod = ext.name
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, CEXTMOD_EXTS)
for files in module_files.values():
for f in files:
yield f
def cython(filename, verbose=False, compile_message=False,
use_cache=False, create_local_c_file=False, annotate=True, sage_namespace=True,
create_local_so_file=False):
r"""
Compile a Cython file. This converts a Cython file to a C (or C++ file),
and then compiles that. The .c file and the .so file are
created in a temporary directory.
INPUT:
- ``filename`` - the name of the file to be compiled. Should end with
'pyx'.
- ``verbose`` (bool, default False) - if True, print debugging
information.
- ``compile_message`` (bool, default False) - if True, print
``'Compiling <filename>...'`` to the standard error.
- ``use_cache`` (bool, default False) - if True, check the
temporary build directory to see if there is already a
corresponding .so file. If so, and if the .so file is newer than the
Cython file, don't recompile, just reuse the .so file.
- ``create_local_c_file`` (bool, default False) - if True, save a
copy of the .c file in the current directory.
- ``annotate`` (bool, default True) - if True, create an html file which
annotates the conversion from .pyx to .c. By default this is only created
in the temporary directory, but if ``create_local_c_file`` is also True,
then save a copy of the .html file in the current directory.
- ``sage_namespace`` (bool, default True) - if True, import
``sage.all``.
- ``create_local_so_file`` (bool, default False) - if True, save a
copy of the compiled .so file in the current directory.
TESTS:
Before :trac:`12975`, it would have beeen needed to write ``#clang c++``,
but upper case ``C++`` has resulted in an error::
sage: code = [
... "#clang C++",
... "#cinclude %s/include/singular %s/include/factory"%(SAGE_LOCAL, SAGE_LOCAL),
... "#clib m readline singular givaro ntl gmpxx gmp",
... "from sage.rings.polynomial.multi_polynomial_libsingular cimport MPolynomial_libsingular",
... "from sage.libs.singular.polynomial cimport singular_polynomial_pow",
... "def test(MPolynomial_libsingular p):",
... " singular_polynomial_pow(&p._poly, p._poly, 2, p._parent_ring)"]
sage: cython(os.linesep.join(code))
The function ``test`` now manipulates internal C data of polynomials,
squaring them::
sage: P.<x,y>=QQ[]
sage: test(x)
sage: x
x^2
Check that compiling c++ code works::
sage: cython("#clang C++\n"+
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
"""
if not filename.endswith('pyx'):
print("Warning: file (={}) should have extension .pyx".format(filename), file=sys.stderr)
# base is the name of the .so module that we create. If we are
# creating a local shared object file, we use a more natural
# naming convention. If we are not creating a local shared object
# file, the main constraint is that it is unique and determined by
# the file that we're running Cython on, so that in some cases we
# can cache the result (e.g., recompiling the same pyx file during
# the same session).
if create_local_so_file:
base, ext = os.path.splitext(os.path.basename(filename))
base = sanitize(base)
else:
base = sanitize(os.path.abspath(filename))
# This is the *temporary* directory where we build the pyx file.
# This is deleted when sage exits, which means pyx files must be
# rebuilt every time Sage is restarted at present.
build_dir = os.path.join(SPYX_TMP, base)
if os.path.exists(build_dir):
# There is already a module here. Maybe we do not have to rebuild?
# Find the name.
if use_cache:
from sage.misc.sageinspect import loadable_module_extension
prev_so = [F for F in os.listdir(build_dir) if F.endswith(loadable_module_extension())]
if len(prev_so) > 0:
prev_so = prev_so[0] # should have length 1 because of deletes below
if os.path.getmtime(filename) <= os.path.getmtime('%s/%s'%(build_dir, prev_so)):
# We do not have to rebuild.
return prev_so[:-len(loadable_module_extension())], build_dir
else:
os.makedirs(build_dir)
for F in os.listdir(build_dir):
G = '%s/%s'%(build_dir,F)
try:
if not os.path.isdir(G):
os.unlink(G)
except OSError:
pass
# Get the absolute path to the directory that contains the pyx file.
# We will use this only to make some convenient symbolic links.
abs_base = os.path.split(os.path.abspath(filename))[0]
# bad things happen if the current directory is SAGE_SRC
if not os.path.exists("%s/sage" % abs_base):
cmd = 'cd "%s"; ln -sf "%s"/* .'%(build_dir, abs_base)
os.system(cmd)
if os.path.exists("%s/setup.py" % build_dir):
os.unlink("%s/setup.py" % build_dir)
if compile_message:
print("Compiling {}...".format(filename), file=sys.stderr)
F = open(filename).read()
F, libs, includes, language, additional_source_files, extra_args, libdirs = pyx_preparse(F)
# add the working directory to the includes so custom headers etc. work
includes.append(os.path.split(os.path.splitext(filename)[0])[0])
if language == 'c++':
extension = "cpp"
else:
extension = "c"
if create_local_so_file:
name = base
else:
global sequence_number
if base not in sequence_number:
sequence_number[base] = 0
name = '%s_%s'%(base, sequence_number[base])
# increment the sequence number so will use a different one next time.
sequence_number[base] += 1
file_list = []
for fname in additional_source_files:
fname = fname.replace("$SAGE_SRC", SAGE_SRC)
fname = fname.replace("$SAGE_LOCAL", SAGE_LOCAL)
if fname.startswith(os.path.sep):
file_list.append("'"+fname+"'")
else:
file_list.append("'"+os.path.abspath(os.curdir)+"/"+fname+"'")
additional_source_files = ",".join(file_list)
pyx = '%s/%s.pyx'%(build_dir, name)
open(pyx,'w').write(F)
setup="""
# Build using 'python setup.py'
import distutils.sysconfig, os, sys
from distutils.core import setup, Extension
from sage.env import SAGE_LOCAL
extra_compile_args = %s
ext_modules = [Extension('%s', sources=['%s.%s', %s],
libraries=%s,
library_dirs=[SAGE_LOCAL + '/lib/'] + %s,
extra_compile_args = extra_compile_args,
language = '%s' )]
setup(ext_modules = ext_modules,
include_dirs = %s)
"""%(extra_args, name, name, extension, additional_source_files, libs, libdirs, language, includes)
open('%s/setup.py'%build_dir,'w').write(setup)
cython_include = ' '.join(["-I '%s'"%x for x in includes if len(x.strip()) > 0 ])
options = ['-p']
if annotate:
options.append('-a')
if sage_namespace:
options.append('--pre-import sage.all')
cmd = "cd '{DIR}' && cython {OPT} {INC} {LANG} '{NAME}.pyx' 1>log 2>err ".format(
DIR=build_dir,
OPT=' '.join(options),
INC=cython_include,
LANG='--cplus' if language=='c++' else '',
NAME=name)
if create_local_c_file:
target_c = '%s/_%s.c'%(os.path.abspath(os.curdir), base)
if language == 'c++':
target_c = target_c + "pp"
cmd += " && cp '%s.c' '%s'"%(name, target_c)
if annotate:
target_html = '%s/_%s.html'%(os.path.abspath(os.curdir), base)
cmd += " && cp '%s.html' '%s'"%(name, target_html)
if verbose:
print(cmd)
if os.system(cmd):
log = open('%s/log'%build_dir).read()
err = subtract_from_line_numbers(open('%s/err'%build_dir).read(), offset)
raise RuntimeError("Error converting {} to C:\n{}\n{}".format(filename, log, err))
cmd = 'cd %s && python setup.py build 1>log 2>err'%build_dir
if verbose:
print(cmd)
if os.system(cmd):
log = open('%s/log'%build_dir).read()
err = open('%s/err'%build_dir).read()
raise RuntimeError("Error compiling {}:\n{}\n{}".format(filename, log, err))
# Move from lib directory.
cmd = 'mv %s/build/lib.*/* %s'%(build_dir, build_dir)
if verbose:
print(cmd)
if os.system(cmd):
raise RuntimeError("Error copying extension module for {}".format(filename))
if create_local_so_file:
# Copy from lib directory into local directory
from sage.misc.sageinspect import loadable_module_extension
cmd = 'cp %s/%s%s %s'%(build_dir, name, loadable_module_extension(), os.path.abspath(os.curdir))
if os.system(cmd):
raise RuntimeError("Error making local copy of shared object library for {}".format(filename))
return name, build_dir
def cython(filename, verbose=0, compile_message=False,
use_cache=False, create_local_c_file=False, annotate=True, sage_namespace=True,
create_local_so_file=False):
r"""
Compile a Cython file. This converts a Cython file to a C (or C++ file),
and then compiles that. The .c file and the .so file are
created in a temporary directory.
INPUT:
- ``filename`` -- the name of the file to be compiled. Should end with
'pyx'.
- ``verbose`` (integer, default 0) -- level of verbosity. A negative
value ensures complete silence.
- ``compile_message`` (bool, default False) -- if True, print
``'Compiling <filename>...'`` to the standard error.
- ``use_cache`` (bool, default False) -- if True, check the
temporary build directory to see if there is already a
corresponding .so file. If so, and if the .so file is newer than the
Cython file, don't recompile, just reuse the .so file.
- ``create_local_c_file`` (bool, default False) -- if True, save a
copy of the ``.c`` or ``.cpp`` file in the current directory.
- ``annotate`` (bool, default True) -- if True, create an html file which
annotates the conversion from .pyx to .c. By default this is only created
in the temporary directory, but if ``create_local_c_file`` is also True,
then save a copy of the .html file in the current directory.
- ``sage_namespace`` (bool, default True) -- if True, import
``sage.all``.
- ``create_local_so_file`` (bool, default False) -- if True, save a
copy of the compiled .so file in the current directory.
OUTPUT: a tuple ``(name, dir)`` where ``name`` is the name
of the compiled module and ``dir`` is the directory containing
the generated files.
TESTS:
Before :trac:`12975`, it would have been needed to write ``#clang c++``,
but upper case ``C++`` has resulted in an error.
Using pkgconfig to find the libraries, headers and macros. This is a
work around while waiting for :trac:`22461` which will offer a better
solution::
sage: code = [
....: "#clang C++",
....: "from sage.rings.polynomial.multi_polynomial_libsingular cimport MPolynomial_libsingular",
....: "from sage.libs.singular.polynomial cimport singular_polynomial_pow",
....: "def test(MPolynomial_libsingular p):",
....: " singular_polynomial_pow(&p._poly, p._poly, 2, p._parent_ring)"]
sage: cython(os.linesep.join(code))
The function ``test`` now manipulates internal C data of polynomials,
squaring them::
sage: P.<x,y>=QQ[]
sage: test(x)
sage: x
x^2
Check that compiling C++ code works::
sage: cython("# distutils: language = c++\n"+
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
Check that compiling C++ code works when creating a local C file,
first moving to a tempdir to avoid clutter. Before :trac:`22113`,
the create_local_c_file argument was not tested for C++ code::
sage: d = sage.misc.temporary_file.tmp_dir()
sage: os.chdir(d)
sage: with open("test.pyx", 'w') as f:
....: _ = f.write("# distutils: language = c++\n"
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
sage: output = sage.misc.cython.cython("test.pyx", create_local_c_file=True)
Accessing a ``.pxd`` file from the current directory works::
sage: d = sage.misc.temporary_file.tmp_dir()
sage: os.chdir(d)
sage: with open("helper.pxd", 'w') as f:
....: _ = f.write("cdef inline int the_answer(): return 42")
sage: cython('''
....: from helper cimport the_answer
....: print(the_answer())
....: ''')
42
Warning and error messages generated by Cython are properly
handled. Warnings are only shown if verbose >= 0::
sage: code = '''
....: def test_unreachable():
....: raise Exception
....: return 42
....: '''
sage: cython(code, verbose=-1)
sage: cython(code, verbose=0)
warning: ...:4:4: Unreachable code
sage: cython("foo = bar\n")
Traceback (most recent call last):
...
RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
foo = bar
^
------------------------------------------------------------
<BLANKLINE>
...:1:6: undeclared name not builtin: bar
sage: cython("cdef extern from 'no_such_header_file': pass")
Traceback (most recent call last):
...
RuntimeError: ...
As of :trac:`29139` the default is ``cdivision=True``::
sage: cython('''
....: cdef size_t foo = 3/2
....: ''')
"""
if not filename.endswith('pyx'):
print("Warning: file (={}) should have extension .pyx".format(filename), file=sys.stderr)
# base is the name of the .so module that we create. If we are
# creating a local shared object file, we use a more natural
# naming convention. If we are not creating a local shared object
# file, the main constraint is that it is unique and determined by
# the file that we're running Cython on, so that in some cases we
# can cache the result (e.g., recompiling the same pyx file during
# the same session).
if create_local_so_file:
base, ext = os.path.splitext(os.path.basename(filename))
else:
base = os.path.abspath(filename)
base = sanitize(base)
# This is the *temporary* directory where we store the pyx file.
# This is deleted when Sage exits, which means pyx files must be
# rebuilt every time Sage is restarted at present.
target_dir = os.path.join(SPYX_TMP, base)
# Build directory for Cython/distutils
build_dir = os.path.join(target_dir, "build")
if os.path.exists(target_dir):
# There is already a module here. Maybe we do not have to rebuild?
# Find the name.
if use_cache:
from sage.misc.sageinspect import loadable_module_extension
prev_so = [F for F in os.listdir(target_dir) if F.endswith(loadable_module_extension())]
if len(prev_so) > 0:
prev_so = prev_so[0] # should have length 1 because of deletes below
if os.path.getmtime(filename) <= os.path.getmtime('%s/%s' % (target_dir, prev_so)):
# We do not have to rebuild.
return prev_so[:-len(loadable_module_extension())], target_dir
# Delete all ordinary files in target_dir
for F in os.listdir(target_dir):
G = os.path.join(target_dir, F)
if os.path.isdir(G):
continue
try:
os.unlink(G)
except OSError:
pass
else:
sage_makedirs(target_dir)
if create_local_so_file:
name = base
else:
global sequence_number
if base not in sequence_number:
sequence_number[base] = 0
name = '%s_%s' % (base, sequence_number[base])
# increment the sequence number so will use a different one next time.
sequence_number[base] += 1
if compile_message:
sys.stderr.write("Compiling {}...\n".format(filename))
sys.stderr.flush()
# Copy original file to the target directory.
pyxfile = os.path.join(target_dir, name + ".pyx")
shutil.copy(filename, pyxfile)
# Add current working directory to includes. This is needed because
# we cythonize from a different directory. See Trac #24764.
standard_libs, standard_libdirs, standard_includes, aliases = _standard_libs_libdirs_incdirs_aliases()
includes = [os.getcwd()] + standard_includes
# Now do the actual build, directly calling Cython and distutils
from Cython.Build import cythonize
from Cython.Compiler.Errors import CompileError
import Cython.Compiler.Options
try:
# Import setuptools before importing distutils, so that setuptools
# can replace distutils by its own vendored copy.
import setuptools
from setuptools.dist import Distribution
from setuptools.extension import Extension
except ImportError:
# Fall back to distutils (stdlib); note that it is deprecated
# in Python 3.10, 3.11; https://www.python.org/dev/peps/pep-0632/
from distutils.dist import Distribution
from distutils.core import Extension
from distutils.log import set_verbosity
set_verbosity(verbose)
Cython.Compiler.Options.annotate = annotate
Cython.Compiler.Options.embed_pos_in_docstring = True
Cython.Compiler.Options.pre_import = "sage.all" if sage_namespace else None
extra_compile_args = ['-w'] # no warnings
extra_link_args = []
if sys.platform == 'cygwin':
# Link using --enable-auto-image-base, reducing the likelihood of the
# new DLL colliding with existing DLLs in memory.
# Note: Cygwin locates --enable-auto-image-base DLLs in the range
# 0x4:00000000 up to 0x6:00000000; this is documented in heap.cc in the
# Cygwin sources, while 0x6:00000000 and up is reserved for the Cygwin
# heap.
# In practice, Sage has enough DLLs that when rebasing everything we
# use up through, approximately, 0x4:80000000 though there is nothing
# precise here. When running 'rebase' it just start from 0x2:00000000
# (below that is reserved for Cygwin's DLL and some internal
# structures).
# Therefore, to minimize the likelihood of collision with one of Sage's
# standard DLLs, while giving ~2GB (should be more than enough) for
# Sage to grow, we base these DLLs from 0x5:8000000, leaving again ~2GB
# for temp DLLs which in normal use should be more than enough.
# See https://trac.sagemath.org/ticket/28258
# It should be noted, this is not a bulletproof solution; there is
# still a potential for DLL overlaps with this. But this reduces the
# probability thereof, especially in normal practice.
dll_filename = os.path.splitext(pyxfile)[0] + '.dll'
image_base = _compute_dll_image_base(dll_filename)
extra_link_args.extend(['-Wl,--disable-auto-image-base',
'-Wl,--image-base=0x{:x}'.format(image_base)])
ext = Extension(name,
sources=[pyxfile],
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
libraries=standard_libs,
library_dirs=standard_libdirs)
directives = dict(language_level=sys.version_info[0], cdivision=True)
try:
# Change directories to target_dir so that Cython produces the correct
# relative path; https://trac.sagemath.org/ticket/24097
with restore_cwd(target_dir):
try:
ext, = cythonize([ext],
aliases=aliases,
include_path=includes,
compiler_directives=directives,
quiet=(verbose <= 0),
errors_to_stderr=False,
use_listing_file=True)
finally:
# Read the "listing file" which is the file containing
# warning and error messages generated by Cython.
try:
with open(name + ".lis") as f:
cython_messages = f.read()
except IOError:
cython_messages = "Error compiling Cython file"
except CompileError:
raise RuntimeError(cython_messages.strip())
if verbose >= 0:
sys.stderr.write(cython_messages)
sys.stderr.flush()
if create_local_c_file:
shutil.copy(os.path.join(target_dir, ext.sources[0]),
os.curdir)
if annotate:
shutil.copy(os.path.join(target_dir, name + ".html"),
os.curdir)
# This emulates running "setup.py build" with the correct options
dist = Distribution()
dist.ext_modules = [ext]
dist.include_dirs = includes
buildcmd = dist.get_command_obj("build")
buildcmd.build_base = build_dir
buildcmd.build_lib = target_dir
try:
# Capture errors from distutils and its child processes
with open(os.path.join(target_dir, name + ".err"), 'w+') as errfile:
try:
# Redirect stderr to errfile. We use the file descriptor
# number "2" instead of "sys.stderr" because we really
# want to redirect the messages from GCC. These are sent
# to the actual stderr, regardless of what sys.stderr is.
sys.stderr.flush()
with redirection(2, errfile, close=False):
dist.run_command("build")
finally:
errfile.seek(0)
distutils_messages = errfile.read()
except Exception as msg:
msg = str(msg) + "\n" + distutils_messages
raise RuntimeError(msg.strip())
if verbose >= 0:
sys.stderr.write(distutils_messages)
sys.stderr.flush()
if create_local_so_file:
# Copy module to current directory
from sage.misc.sageinspect import loadable_module_extension
shutil.copy(os.path.join(target_dir, name + loadable_module_extension()),
os.curdir)
return name, target_dir
def cython(filename, verbose=0, compile_message=False,
use_cache=False, create_local_c_file=False, annotate=True, sage_namespace=True,
create_local_so_file=False):
r"""
Compile a Cython file. This converts a Cython file to a C (or C++ file),
and then compiles that. The .c file and the .so file are
created in a temporary directory.
INPUT:
- ``filename`` -- the name of the file to be compiled. Should end with
'pyx'.
- ``verbose`` (integer, default 0) -- level of verbosity. A negative
value ensures complete silence.
- ``compile_message`` (bool, default False) -- if True, print
``'Compiling <filename>...'`` to the standard error.
- ``use_cache`` (bool, default False) -- if True, check the
temporary build directory to see if there is already a
corresponding .so file. If so, and if the .so file is newer than the
Cython file, don't recompile, just reuse the .so file.
- ``create_local_c_file`` (bool, default False) -- if True, save a
copy of the ``.c`` or ``.cpp`` file in the current directory.
- ``annotate`` (bool, default True) -- if True, create an html file which
annotates the conversion from .pyx to .c. By default this is only created
in the temporary directory, but if ``create_local_c_file`` is also True,
then save a copy of the .html file in the current directory.
- ``sage_namespace`` (bool, default True) -- if True, import
``sage.all``.
- ``create_local_so_file`` (bool, default False) -- if True, save a
copy of the compiled .so file in the current directory.
OUTPUT: a tuple ``(name, dir)`` where ``name`` is the name
of the compiled module and ``dir`` is the directory containing
the generated files.
TESTS:
Before :trac:`12975`, it would have been needed to write ``#clang c++``,
but upper case ``C++`` has resulted in an error.
Using pkgconfig to find the libraries, headers and macros. This is a
work around while waiting for :trac:`22461` which will offer a better
solution::
sage: code = [
....: "#clang C++",
....: "from sage.rings.polynomial.multi_polynomial_libsingular cimport MPolynomial_libsingular",
....: "from sage.libs.singular.polynomial cimport singular_polynomial_pow",
....: "def test(MPolynomial_libsingular p):",
....: " singular_polynomial_pow(&p._poly, p._poly, 2, p._parent_ring)"]
sage: cython(os.linesep.join(code))
The function ``test`` now manipulates internal C data of polynomials,
squaring them::
sage: P.<x,y>=QQ[]
sage: test(x)
sage: x
x^2
Check that compiling C++ code works::
sage: cython("# distutils: language = c++\n"+
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
Check that compiling C++ code works when creating a local C file,
first moving to a tempdir to avoid clutter. Before :trac:`22113`,
the create_local_c_file argument was not tested for C++ code::
sage: import sage.misc.cython
sage: d = sage.misc.temporary_file.tmp_dir()
sage: os.chdir(d)
sage: with open("test.pyx", 'w') as f:
....: _ = f.write("# distutils: language = c++\n"
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
sage: output = sage.misc.cython.cython("test.pyx", create_local_c_file=True)
Accessing a ``.pxd`` file from the current directory works::
sage: import sage.misc.cython
sage: d = sage.misc.temporary_file.tmp_dir()
sage: os.chdir(d)
sage: with open("helper.pxd", 'w') as f:
....: f.write("cdef inline int the_answer(): return 42")
sage: cython('''
....: from helper cimport the_answer
....: print(the_answer())
....: ''')
42
Warning and error messages generated by Cython are properly
handled. Warnings are only shown if verbose >= 0::
sage: code = '''
....: def test_unreachable():
....: raise Exception
....: return 42
....: '''
sage: cython(code, verbose=-1)
sage: cython(code, verbose=0)
warning: ...:4:4: Unreachable code
sage: cython("foo = bar\n")
Traceback (most recent call last):
...
RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
foo = bar
^
------------------------------------------------------------
<BLANKLINE>
...:1:6: undeclared name not builtin: bar
sage: cython("cdef extern from 'no_such_header_file': pass")
Traceback (most recent call last):
...
RuntimeError: ...
Sage used to automatically include various ``.pxi`` files. Since
:trac:`22805`, we no longer do this. But we make sure to give a
useful message in case the ``.pxi`` files were needed::
sage: cython("sig_malloc(0)\n")
Traceback (most recent call last):
...
RuntimeError: Error compiling Cython file:
------------------------------------------------------------
...
sig_malloc(0)
^
------------------------------------------------------------
<BLANKLINE>
...:1:0: undeclared name not builtin: sig_malloc
<BLANKLINE>
NOTE: Sage no longer automatically includes the deprecated files
"cdefs.pxi", "signals.pxi" and "stdsage.pxi" in Cython files.
You can fix your code by adding "from cysignals.memory cimport sig_malloc".
"""
if not filename.endswith('pyx'):
print("Warning: file (={}) should have extension .pyx".format(filename), file=sys.stderr)
# base is the name of the .so module that we create. If we are
# creating a local shared object file, we use a more natural
# naming convention. If we are not creating a local shared object
# file, the main constraint is that it is unique and determined by
# the file that we're running Cython on, so that in some cases we
# can cache the result (e.g., recompiling the same pyx file during
# the same session).
if create_local_so_file:
base, ext = os.path.splitext(os.path.basename(filename))
else:
base = os.path.abspath(filename)
base = sanitize(base)
# This is the *temporary* directory where we store the pyx file.
# This is deleted when Sage exits, which means pyx files must be
# rebuilt every time Sage is restarted at present.
target_dir = os.path.join(SPYX_TMP, base)
# Build directory for Cython/distutils
build_dir = os.path.join(target_dir, "build")
if os.path.exists(target_dir):
# There is already a module here. Maybe we do not have to rebuild?
# Find the name.
if use_cache:
from sage.misc.sageinspect import loadable_module_extension
prev_so = [F for F in os.listdir(target_dir) if F.endswith(loadable_module_extension())]
if len(prev_so) > 0:
prev_so = prev_so[0] # should have length 1 because of deletes below
if os.path.getmtime(filename) <= os.path.getmtime('%s/%s'%(target_dir, prev_so)):
# We do not have to rebuild.
return prev_so[:-len(loadable_module_extension())], target_dir
# Delete all ordinary files in target_dir
for F in os.listdir(target_dir):
G = os.path.join(target_dir, F)
if os.path.isdir(G):
continue
try:
os.unlink(G)
except OSError:
pass
else:
sage_makedirs(target_dir)
if create_local_so_file:
name = base
else:
global sequence_number
if base not in sequence_number:
sequence_number[base] = 0
name = '%s_%s'%(base, sequence_number[base])
# increment the sequence number so will use a different one next time.
sequence_number[base] += 1
if compile_message:
print("Compiling {}...".format(filename), file=sys.stderr)
sys.stderr.flush()
with open(filename) as f:
(preparsed, libs, includes, language, additional_source_files,
extra_args, libdirs) = _pyx_preparse(f.read())
# New filename with preparsed code.
# NOTE: if we ever stop preparsing, we should still copy the
# original file to the target directory.
pyxfile = os.path.join(target_dir, name + ".pyx")
with open(pyxfile, 'w') as f:
f.write(preparsed)
extra_sources = []
for fname in additional_source_files:
fname = fname.replace("$SAGE_SRC", SAGE_SRC)
fname = fname.replace("$SAGE_LOCAL", SAGE_LOCAL)
extra_sources.append(fname)
# Add current working directory to includes. This is needed because
# we cythonize from a different directory. See Trac #24764.
includes.insert(0, os.getcwd())
# Now do the actual build, directly calling Cython and distutils
from Cython.Build import cythonize
from Cython.Compiler.Errors import CompileError
import Cython.Compiler.Options
from distutils.dist import Distribution
from distutils.core import Extension
from distutils.log import set_verbosity
set_verbosity(verbose)
Cython.Compiler.Options.annotate = annotate
Cython.Compiler.Options.embed_pos_in_docstring = True
Cython.Compiler.Options.pre_import = "sage.all" if sage_namespace else None
ext = Extension(name,
sources=[pyxfile] + extra_sources,
libraries=libs,
library_dirs=[os.path.join(SAGE_LOCAL, "lib")] + libdirs,
extra_compile_args=extra_args,
language=language)
try:
# Change directories to target_dir so that Cython produces the correct
# relative path; https://trac.sagemath.org/ticket/24097
with restore_cwd(target_dir):
try:
ext, = cythonize([ext],
aliases=cython_aliases(),
include_path=includes,
quiet=(verbose <= 0),
errors_to_stderr=False,
use_listing_file=True)
finally:
# Read the "listing file" which is the file containing
# warning and error messages generated by Cython.
try:
with open(name + ".lis") as f:
cython_messages = f.read()
except IOError:
cython_messages = "Error compiling Cython file"
except CompileError:
# Check for names in old_pxi_names
for pxd, names in old_pxi_names.items():
for name in names:
if re.search(r"\b{}\b".format(name), cython_messages):
cython_messages += dedent(
"""
NOTE: Sage no longer automatically includes the deprecated files
"cdefs.pxi", "signals.pxi" and "stdsage.pxi" in Cython files.
You can fix your code by adding "from {} cimport {}".
""".format(pxd, name))
raise RuntimeError(cython_messages.strip())
if verbose >= 0:
sys.stderr.write(cython_messages)
sys.stderr.flush()
if create_local_c_file:
shutil.copy(os.path.join(target_dir, ext.sources[0]),
os.curdir)
if annotate:
shutil.copy(os.path.join(target_dir, name + ".html"),
os.curdir)
# This emulates running "setup.py build" with the correct options
dist = Distribution()
dist.ext_modules = [ext]
dist.include_dirs = includes
buildcmd = dist.get_command_obj("build")
buildcmd.build_base = build_dir
buildcmd.build_lib = target_dir
try:
# Capture errors from distutils and its child processes
with open(os.path.join(target_dir, name + ".err"), 'w+') as errfile:
try:
# Redirect stderr to errfile. We use the file descriptor
# number "2" instead of "sys.stderr" because we really
# want to redirect the messages from GCC. These are sent
# to the actual stderr, regardless of what sys.stderr is.
sys.stderr.flush()
with redirection(2, errfile, close=False):
dist.run_command("build")
finally:
errfile.seek(0)
distutils_messages = errfile.read()
except Exception as msg:
msg = str(msg) + "\n" + distutils_messages
raise RuntimeError(msg.strip())
if verbose >= 0:
sys.stderr.write(distutils_messages)
sys.stderr.flush()
if create_local_so_file:
# Copy module to current directory
from sage.misc.sageinspect import loadable_module_extension
shutil.copy(os.path.join(target_dir, name + loadable_module_extension()),
os.curdir)
return name, target_dir
def _find_stale_files(site_packages, python_packages, python_modules, ext_modules, data_files):
"""
Find stale files
This method lists all files installed and then subtracts the ones
which are intentionally being installed.
EXAMPLES:
It is crucial that only truly stale files are being found, of
course. We check that when the doctest is being run, that is,
after installation, there are no stale files::
sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_CYTHONIZED
sage: from sage_setup.find import find_python_sources, find_extra_files
sage: python_packages, python_modules = find_python_sources(
....: SAGE_SRC, ['sage', 'sage_setup'])
sage: extra_files = find_extra_files(python_packages, SAGE_SRC,
....: SAGE_CYTHONIZED, SAGE_LIB, ["ntlwrap.cpp"])
sage: from sage_setup.clean import _find_stale_files
TODO: move ``module_list.py`` into ``sage_setup`` and also check
extension modules::
sage: stale_iter = _find_stale_files(SAGE_LIB, python_packages, python_modules, [], extra_files)
sage: from sage.misc.sageinspect import loadable_module_extension
sage: skip_extensions = (loadable_module_extension(),)
sage: for f in stale_iter:
....: if f.endswith(skip_extensions): continue
....: print('Found stale file: ' + f)
"""
from sage.misc.sageinspect import loadable_module_extension
PYMOD_EXTS = (os.path.extsep + 'py', os.path.extsep + 'pyc')
CEXTMOD_EXTS = (loadable_module_extension(),)
INIT_FILES = tuple('__init__' + x for x in PYMOD_EXTS)
module_files = installed_files_by_module(site_packages, ['sage', 'sage_setup'])
for mod in python_packages:
try:
files = module_files[mod]
except KeyError:
# the source module "mod" has not been previously installed, fine.
continue
_remove(files, mod, INIT_FILES)
for mod in python_modules:
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, PYMOD_EXTS)
for ext in ext_modules:
mod = ext.name
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, CEXTMOD_EXTS)
# Convert data_files to a set
installed_files = set()
for dir, files in data_files:
dir = os.path.relpath(dir, site_packages)
if dir.startswith("."): # dir is not inside site_packages
continue
for f in files:
installed_files.add(os.path.join(dir, os.path.basename(f)))
for files in module_files.values():
for f in files:
if f not in installed_files:
yield f
def cython(filename,
verbose=0,
compile_message=False,
use_cache=False,
create_local_c_file=False,
annotate=True,
sage_namespace=True,
create_local_so_file=False):
r"""
Compile a Cython file. This converts a Cython file to a C (or C++ file),
and then compiles that. The .c file and the .so file are
created in a temporary directory.
INPUT:
- ``filename`` -- the name of the file to be compiled. Should end with
'pyx'.
- ``verbose`` (integer, default 0) -- level of verbosity.
- ``compile_message`` (bool, default False) -- if True, print
``'Compiling <filename>...'`` to the standard error.
- ``use_cache`` (bool, default False) -- if True, check the
temporary build directory to see if there is already a
corresponding .so file. If so, and if the .so file is newer than the
Cython file, don't recompile, just reuse the .so file.
- ``create_local_c_file`` (bool, default False) -- if True, save a
copy of the ``.c`` or ``.cpp`` file in the current directory.
- ``annotate`` (bool, default True) -- if True, create an html file which
annotates the conversion from .pyx to .c. By default this is only created
in the temporary directory, but if ``create_local_c_file`` is also True,
then save a copy of the .html file in the current directory.
- ``sage_namespace`` (bool, default True) -- if True, import
``sage.all``.
- ``create_local_so_file`` (bool, default False) -- if True, save a
copy of the compiled .so file in the current directory.
TESTS:
Before :trac:`12975`, it would have been needed to write ``#clang c++``,
but upper case ``C++`` has resulted in an error.
Using pkgconfig to find the libraries, headers and macros. This is a
work around while waiting for :trac:`22461` which will offer a better
solution::
sage: code = [
....: "#clang C++",
....: "from sage.rings.polynomial.multi_polynomial_libsingular cimport MPolynomial_libsingular",
....: "from sage.libs.singular.polynomial cimport singular_polynomial_pow",
....: "def test(MPolynomial_libsingular p):",
....: " singular_polynomial_pow(&p._poly, p._poly, 2, p._parent_ring)"]
sage: cython(os.linesep.join(code))
The function ``test`` now manipulates internal C data of polynomials,
squaring them::
sage: P.<x,y>=QQ[]
sage: test(x)
sage: x
x^2
Check that compiling C++ code works::
sage: cython("# distutils: language = c++\n"+
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
Check that compiling C++ code works when creating a local C file,
first moving to a tempdir to avoid clutter. Before :trac:`22113`,
the create_local_c_file argument was not tested for C++ code::
sage: import sage.misc.cython
sage: d = sage.misc.temporary_file.tmp_dir()
sage: os.chdir(d)
sage: with open("test.pyx", 'w') as f:
....: _ = f.write("# distutils: language = c++\n"
....: "from libcpp.vector cimport vector\n"
....: "cdef vector[int] * v = new vector[int](4)\n")
sage: output = sage.misc.cython.cython("test.pyx", create_local_c_file=True)
Sage used to automatically include various ``.pxi`` files. Since
:trac:`22805`, we no longer do this. But we make sure to give a
useful message in case the ``.pxi`` files were needed::
sage: cython("sig_malloc(0)")
Traceback (most recent call last):
...
RuntimeError: Error converting ... to C
NOTE: Sage no longer automatically includes the deprecated files
"cdefs.pxi", "signals.pxi" and "stdsage.pxi" in Cython files.
You can fix your code by adding "from cysignals.memory cimport sig_malloc".
"""
if not filename.endswith('pyx'):
print(
"Warning: file (={}) should have extension .pyx".format(filename),
file=sys.stderr)
# base is the name of the .so module that we create. If we are
# creating a local shared object file, we use a more natural
# naming convention. If we are not creating a local shared object
# file, the main constraint is that it is unique and determined by
# the file that we're running Cython on, so that in some cases we
# can cache the result (e.g., recompiling the same pyx file during
# the same session).
if create_local_so_file:
base, ext = os.path.splitext(os.path.basename(filename))
base = sanitize(base)
else:
base = sanitize(os.path.abspath(filename))
# This is the *temporary* directory where we store the pyx file.
# This is deleted when Sage exits, which means pyx files must be
# rebuilt every time Sage is restarted at present.
target_dir = os.path.join(SPYX_TMP, base)
# Build directory for Cython/distutils
build_dir = os.path.join(target_dir, "build")
if os.path.exists(target_dir):
# There is already a module here. Maybe we do not have to rebuild?
# Find the name.
if use_cache:
from sage.misc.sageinspect import loadable_module_extension
prev_so = [
F for F in os.listdir(target_dir)
if F.endswith(loadable_module_extension())
]
if len(prev_so) > 0:
prev_so = prev_so[
0] # should have length 1 because of deletes below
if os.path.getmtime(filename) <= os.path.getmtime(
'%s/%s' % (target_dir, prev_so)):
# We do not have to rebuild.
return prev_so[:-len(loadable_module_extension()
)], target_dir
# Delete all ordinary files in target_dir
for F in os.listdir(target_dir):
G = os.path.join(target_dir, F)
if os.path.isdir(G):
continue
try:
os.unlink(G)
except OSError:
pass
else:
sage_makedirs(target_dir)
if create_local_so_file:
name = base
else:
global sequence_number
if base not in sequence_number:
sequence_number[base] = 0
name = '%s_%s' % (base, sequence_number[base])
# increment the sequence number so will use a different one next time.
sequence_number[base] += 1
if compile_message:
print("Compiling {}...".format(filename), file=sys.stderr)
with open(filename) as f:
(preparsed, libs, includes, language, additional_source_files,
extra_args, libdirs) = _pyx_preparse(f.read())
# New filename with preparsed code.
# NOTE: if we ever stop preparsing, we should still copy the
# original file to the target directory.
pyxfile = os.path.join(target_dir, name + ".pyx")
with open(pyxfile, 'w') as f:
f.write(preparsed)
extra_sources = []
for fname in additional_source_files:
fname = fname.replace("$SAGE_SRC", SAGE_SRC)
fname = fname.replace("$SAGE_LOCAL", SAGE_LOCAL)
extra_sources.append(fname)
# Now do the actual build, directly calling Cython and distutils
from Cython.Build import cythonize
from Cython.Compiler.Errors import CompileError
import Cython.Compiler.Options
from distutils.dist import Distribution
from distutils.core import Extension
from distutils.log import set_verbosity
set_verbosity(verbose)
Cython.Compiler.Options.annotate = annotate
Cython.Compiler.Options.embed_pos_in_docstring = True
Cython.Compiler.Options.pre_import = "sage.all" if sage_namespace else None
ext = Extension(name,
sources=[pyxfile] + extra_sources,
libraries=libs,
library_dirs=[os.path.join(SAGE_LOCAL, "lib")] + libdirs,
extra_compile_args=extra_args,
language=language)
orig_cwd = os.getcwd()
try:
# Change directories to target_dir so that Cython produces the correct
# relative path; https://trac.sagemath.org/ticket/24097
os.chdir(target_dir)
ext, = cythonize([ext],
aliases=cython_aliases(),
include_path=includes,
quiet=not verbose)
except CompileError:
# Check for names in old_pxi_names
note = ''
for pxd, names in old_pxi_names.items():
for name in names:
if re.search(r"\b{}\b".format(name), preparsed):
note += dedent("""
NOTE: Sage no longer automatically includes the deprecated files
"cdefs.pxi", "signals.pxi" and "stdsage.pxi" in Cython files.
You can fix your code by adding "from {} cimport {}".
""".format(pxd, name))
raise RuntimeError("Error converting {} to C".format(filename) + note)
finally:
os.chdir(orig_cwd)
if create_local_c_file:
shutil.copy(os.path.join(target_dir, ext.sources[0]), os.curdir)
if annotate:
shutil.copy(os.path.join(target_dir, name + ".html"), os.curdir)
# This emulates running "setup.py build" with the correct options
dist = Distribution()
dist.ext_modules = [ext]
dist.include_dirs = includes
buildcmd = dist.get_command_obj("build")
buildcmd.build_base = build_dir
buildcmd.build_lib = target_dir
dist.run_command("build")
if create_local_so_file:
# Copy module to current directory
from sage.misc.sageinspect import loadable_module_extension
shutil.copy(
os.path.join(target_dir, name + loadable_module_extension()),
os.curdir)
return name, target_dir
def _find_stale_files(site_packages, python_packages, python_modules,
ext_modules, data_files):
"""
Find stale files
This method lists all files installed and then subtracts the ones
which are intentionally being installed.
EXAMPLES:
It is crucial that only truly stale files are being found, of
course. We check that when the doctest is being run, that is,
after installation, there are no stale files::
sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_CYTHONIZED
sage: from sage_setup.find import find_python_sources, find_extra_files
sage: python_packages, python_modules = find_python_sources(
....: SAGE_SRC, ['sage', 'sage_setup'])
sage: extra_files = find_extra_files(python_packages, SAGE_SRC,
....: SAGE_CYTHONIZED, SAGE_LIB, ["ntlwrap.cpp"])
sage: from sage_setup.clean import _find_stale_files
TODO: move ``module_list.py`` into ``sage_setup`` and also check
extension modules::
sage: stale_iter = _find_stale_files(SAGE_LIB, python_packages, python_modules, [], extra_files)
sage: from sage.misc.sageinspect import loadable_module_extension
sage: skip_extensions = (loadable_module_extension(),)
sage: for f in stale_iter:
....: if f.endswith(skip_extensions): continue
....: print('Found stale file: ' + f)
"""
from sage.misc.sageinspect import loadable_module_extension
PYMOD_EXTS = (os.path.extsep + 'py', os.path.extsep + 'pyc')
CEXTMOD_EXTS = (loadable_module_extension(), )
INIT_FILES = tuple('__init__' + x for x in PYMOD_EXTS)
module_files = installed_files_by_module(site_packages,
['sage', 'sage_setup'])
for mod in python_packages:
try:
files = module_files[mod]
except KeyError:
# the source module "mod" has not been previously installed, fine.
continue
_remove(files, mod, INIT_FILES)
for mod in python_modules:
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, PYMOD_EXTS)
for ext in ext_modules:
mod = ext.name
try:
files = module_files[mod]
except KeyError:
continue
_remove(files, mod, CEXTMOD_EXTS)
# Convert data_files to a set
installed_files = set()
for dir, files in data_files:
dir = os.path.relpath(dir, site_packages)
if dir.startswith("."): # dir is not inside site_packages
continue
for f in files:
installed_files.add(os.path.join(dir, os.path.basename(f)))
for files in module_files.values():
for f in files:
if f not in installed_files:
yield f
