def test_simple_run(self):
pkg_dir, dist = self.create_dist()
cmd = clean(dist)
# let's add some elements clean should remove
dirs = [(d, os.path.join(pkg_dir, d))
for d in ('build_temp', 'build_lib', 'bdist_base',
'build_scripts', 'build_base')]
for name, path in dirs:
os.mkdir(path)
setattr(cmd, name, path)
if name == 'build_base':
continue
for f in ('one', 'two', 'three'):
self.write_file(os.path.join(path, f))
# let's run the command
cmd.all = 1
cmd.ensure_finalized()
cmd.run()
# make sure the files where removed
for name, path in dirs:
self.assertTrue(not os.path.exists(path),
'%s was not removed' % path)
# let's run the command again (should spit warnings but succeed)
cmd.all = 1
cmd.ensure_finalized()
cmd.run()
def test_simple_run(self):
pkg_dir, dist = self.create_dist()
cmd = clean(dist)
dirs = [ (d, os.path.join(pkg_dir, d)) for d in ('build_temp', 'build_lib', 'bdist_base', 'build_scripts', 'build_base') ]
for name, path in dirs:
os.mkdir(path)
setattr(cmd, name, path)
if name == 'build_base':
continue
for f in ('one', 'two', 'three'):
self.write_file(os.path.join(path, f))
cmd.all = 1
cmd.ensure_finalized()
cmd.run()
for name, path in dirs:
self.assertFalse(os.path.exists(path), '%s was not removed' % path)
cmd.all = 1
cmd.ensure_finalized()
cmd.run()
def run(self):
# invoke the default clean()
import shutil
from distutils.command.clean import clean
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
pycache_folders = [root for root, _, _ in os.walk(working_dir) if root.endswith('__pycache__')]
custom_folders = [folder for cmd in custom_commands.values() for folder in cmd.clean_folders()]
# additional cleanup
for folder in pycache_folders + custom_folders:
try:
if os.path.isfile(folder):
os.remove(folder)
else:
shutil.rmtree(folder)
except:
pass
def run(self):
install.run(self)
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
def compile_c_extension(
generated_source_path: str,
build_dir: Optional[str] = None,
verbose: bool = False,
keep_asserts: bool = True,
disable_optimization:
bool = True, # Significant test_peg_generator speedup.
) -> str:
"""Compile the generated source for a parser generator into an extension module.
The extension module will be generated in the same directory as the provided path
for the generated source, with the same basename (in addition to extension module
metadata). For example, for the source mydir/parser.c the generated extension
in a darwin system with python 3.8 will be mydir/parser.cpython-38-darwin.so.
If *build_dir* is provided, that path will be used as the temporary build directory
of distutils (this is useful in case you want to use a temporary directory).
"""
import distutils.log
from distutils.command.build_ext import build_ext # type: ignore
from distutils.command.clean import clean # type: ignore
from distutils.core import Distribution, Extension
from distutils.tests.support import fixup_build_ext # type: ignore
if verbose:
distutils.log.set_verbosity(distutils.log.DEBUG)
source_file_path = pathlib.Path(generated_source_path)
extension_name = source_file_path.stem
extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
# Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
extra_compile_args.append("-D_Py_TEST_PEGEN")
extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
if keep_asserts:
extra_compile_args.append("-UNDEBUG")
if disable_optimization:
if sys.platform == 'win32':
extra_compile_args.append("/Od")
extra_link_args.append("/LTCG:OFF")
else:
extra_compile_args.append("-O0")
if sysconfig.get_config_var("GNULD") == "yes":
extra_link_args.append("-fno-lto")
extension = [
Extension(
extension_name,
sources=[
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
str(MOD_DIR.parent.parent.parent / "Parser" /
"pegen_errors.c"),
str(MOD_DIR.parent.parent.parent / "Parser" /
"action_helpers.c"),
str(MOD_DIR.parent.parent.parent / "Parser" /
"string_parser.c"),
str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
generated_source_path,
],
include_dirs=[
str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
str(MOD_DIR.parent.parent.parent / "Parser"),
],
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
)
]
dist = Distribution({"name": extension_name, "ext_modules": extension})
cmd = build_ext(dist)
fixup_build_ext(cmd)
cmd.inplace = True
if build_dir:
cmd.build_temp = build_dir
cmd.build_lib = build_dir
cmd.ensure_finalized()
cmd.run()
extension_path = source_file_path.parent / cmd.get_ext_filename(
extension_name)
shutil.move(cmd.get_ext_fullpath(extension_name), extension_path)
cmd = clean(dist)
cmd.finalize_options()
cmd.run()
return extension_path
def compile_c_extension(
generated_source_path: str,
build_dir: Optional[str] = None,
verbose: bool = False,
keep_asserts: bool = True,
) -> str:
"""Compile the generated source for a parser generator into an extension module.
The extension module will be generated in the same directory as the provided path
for the generated source, with the same basename (in addition to extension module
metadata). For example, for the source mydir/parser.c the generated extension
in a darwin system with python 3.8 will be mydir/parser.cpython-38-darwin.so.
If *build_dir* is provided, that path will be used as the temporary build directory
of distutils (this is useful in case you want to use a temporary directory).
"""
import distutils.log
from distutils.core import Distribution, Extension
from distutils.command.clean import clean # type: ignore
from distutils.command.build_ext import build_ext # type: ignore
from distutils.tests.support import fixup_build_ext
if verbose:
distutils.log.set_verbosity(distutils.log.DEBUG)
source_file_path = pathlib.Path(generated_source_path)
extension_name = source_file_path.stem
extra_compile_args = get_extra_flags('CFLAGS', 'PY_CFLAGS_NODIST')
extra_link_args = get_extra_flags('LDFLAGS', 'PY_LDFLAGS_NODIST')
if keep_asserts:
extra_compile_args.append("-UNDEBUG")
extension = [
Extension(
extension_name,
sources=[
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen" /
"pegen.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen" /
"parse_string.c"),
str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
generated_source_path,
],
include_dirs=[
str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
str(MOD_DIR.parent.parent.parent / "Parser"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen"),
],
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
)
]
dist = Distribution({"name": extension_name, "ext_modules": extension})
cmd = build_ext(dist)
fixup_build_ext(cmd)
cmd.inplace = True
if build_dir:
cmd.build_temp = build_dir
cmd.build_lib = build_dir
cmd.ensure_finalized()
cmd.run()
extension_path = source_file_path.parent / cmd.get_ext_filename(
extension_name)
shutil.move(cmd.get_ext_fullpath(extension_name), extension_path)
cmd = clean(dist)
cmd.finalize_options()
cmd.run()
return extension_path
def run(self):
install.run(self)
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
def run(self):
## Setup files.
python_path = os.path.abspath(find_executable("python"))
python_bin_folder = os.path.dirname(python_path)
python_env = os.path.dirname(python_bin_folder)
python_inc = get_python_inc()
numpy_inc = get_include()
if platform_type == "Darwin":
lib_extension = "so"
path_makefile_inc = os.path.join(here, "example_makefiles",
"makefile.inc.Mac.brew")
shutil.copy(path_makefile_inc, makefile)
elif platform_type == "Linux":
gpp_compiler = os.path.abspath(find_executable("g++"))
gcc_compiler = os.path.abspath(find_executable("gcc"))
lib_extension = "so"
## Creating include file.
with open(makefile, 'w') as outfile:
makefile_str = """
### Automatically created make include file.
CC={}
CXX={}
CFLAGS=-fPIC -m64 -Wall -g -O3 -mavx -msse4 -mpopcnt -fopenmp -Wno-sign-compare -fopenmp
CXXFLAGS=$(CFLAGS) -std=c++11
LDFLAGS=-g -fPIC -fopenmp
# common linux flags
SHAREDEXT=so
SHAREDFLAGS=-shared
FAISSSHAREDFLAGS=-shared
""".format(gcc_compiler, gpp_compiler)
outfile.write(makefile_str)
outfile.write("MKLROOT={}\n".format(python_env))
outfile.write(
"BLASLDFLAGS=-Wl,--no-as-needed -L$(MKLROOT)/lib -lmkl_intel_ilp64 -lmkl_core -lmkl_gnu_thread -ldl -lpthread\n"
)
outfile.write("BLASCFLAGS=-DFINTEGER=long\n\n\n")
outfile.write("SWIGEXEC=swig\n")
outfile.write("PYTHONCFLAGS=-I{} -I{}\n".format(
python_inc, numpy_inc))
outfile.close()
else:
print("Unknown platform {}".format(platform_type))
## Run make
os.system("make")
os.system("make py")
## Checking compilation.
check_fpath = os.path.join("python",
"_swigfaiss.{}".format(lib_extension))
if not os.path.exists(check_fpath):
print("Could not find {}".format(check_fpath))
print("Have you run `make` and `make py` "
"(and optionally `cd gpu && make && make py && cd ..`)?")
# make the faiss python package dir
shutil.rmtree("faiss", ignore_errors=True)
os.mkdir("faiss")
shutil.copyfile("python/__init__.py", "faiss/__init__.py")
shutil.copyfile("faiss.py", "faiss/faiss.py")
shutil.copyfile("python/swigfaiss.py", "faiss/swigfaiss.py")
shutil.copyfile(
os.path.join("python", "_swigfaiss.{}".format(lib_extension)),
os.path.join("faiss", "_swigfaiss.{}".format(lib_extension)))
try:
shutil.copyfile("python/_swigfaiss_gpu.so",
"faiss/_swigfaiss_gpu.so")
shutil.copyfile("python/swigfaiss_gpu.py",
"faiss/swigfaiss_gpu.py")
except:
pass
install.run(self)
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
def run(self):
super(CleanInstall, self).run()
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
def run(self):
c = clean(self.distribution)
c.all = True
c.finalize_options()
c.run()
develop.run(self)
def clean_build(self, distribution):
clean_command = clean.clean(distribution)
clean_command.all = True
clean_command.finalize_options()
clean_command.run()