def _get_libs_to_copy(self):
if self._is_unix():
return pyarrow.get_libraries() + \
['arrow_flight', 'arrow_boost_regex', 'arrow_boost_system', 'arrow_boost_filesystem']
elif platform == 'win32':
return pyarrow.get_libraries() + ['arrow_flight']
else:
raise RuntimeError(
'Building on platform {} is not supported yet.'.format(
platform))
def build_extensions(self):
opts = ["-std=c++11", "-g"]
if TILEDBVCF_DEBUG_BUILD:
opts.extend(["-O0"])
else:
opts.extend(["-O2"])
link_opts = []
for ext in self.extensions:
ext.extra_compile_args = opts
ext.extra_link_args = link_opts
import pyarrow
# unversioned symlinks to arrow libraries are required for wheels
# https://github.com/apache/arrow/blob/master/docs/source/python/extending.rst#building-extensions-against-pypi-wheels
pyarrow.create_library_symlinks()
ext.libraries.extend(pyarrow.get_libraries())
ext.include_dirs.append(pyarrow.get_include())
# don't overlink the arrow core library
if "arrow" in ext.libraries:
ext.libraries.remove("arrow")
ext.library_dirs.extend(pyarrow.get_library_dirs())
find_or_build_libtiledbvcf(self)
build_ext.build_extensions(self)
def _get_arrow_lib_as_linker_input(self):
arrow_lib = pyarrow.get_libraries()
link_lib = []
for lib in arrow_lib:
lib_pattern = self._get_pyarrow_lib_pattern(lib)
source = glob.glob(lib_pattern)[0]
link_lib.append(source)
return link_lib
def build_extensions(self):
opts = ['-std=c++11', '-g', '-O2']
link_opts = []
for ext in self.extensions:
ext.extra_compile_args = opts
ext.extra_link_args = link_opts
import pyarrow
ext.include_dirs.append(pyarrow.get_include())
ext.libraries.extend(pyarrow.get_libraries())
ext.library_dirs.extend(pyarrow.get_library_dirs())
find_or_build_libtiledbvcf(self)
build_ext.build_extensions(self)
def gen_gis_core_modules():
gis_core_modules = cythonize(
Extension(name="arctern.arctern_core_",
sources=["arctern/cython/arctern_core_.pyx"]))
for ext in gis_core_modules:
# The Numpy C headers are currently required
ext.include_dirs.append(np.get_include())
ext.include_dirs.append(pa.get_include())
ext.libraries.extend(['arctern'] + pa.get_libraries())
ext.library_dirs.extend(pa.get_library_dirs())
if os.name == 'posix':
ext.extra_compile_args.append('-std=c++11')
# Try uncommenting the following line on Linux
# if you get weird linker errors or runtime crashes
#ext.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0"))
return gis_core_modules
def finalize_options(self):
import pybind11
import pyarrow as pa
build_ext.finalize_options(self)
if not hasattr(self, 'include_dirs'):
self.include_dirs = []
self.include_dirs.append("pybnesian/")
self.include_dirs.append("lib/libfort")
self.include_dirs.append(pybind11.get_include())
if not hasattr(self, 'libraries'):
self.libraries = []
if sys.platform != 'darwin':
self.libraries.append("OpenCL")
self.libraries.extend(pa.get_libraries())
self.libraries.append("nlopt")
if not hasattr(self, 'library_dirs'):
self.library_dirs = []
self.library_dirs.extend(pa.get_library_dirs())
if sys.platform == "win32":
if "CL_LIBRARY_PATH" in os.environ:
cl_library_path = os.environ["CL_LIBRARY_PATH"]
else:
cl_library_path = find_opencl.find_opencl_library_dir()
if cl_library_path is None:
raise RuntimeError("OpenCL library path not found. Set \"CL_LIBRARY_PATH\" environment variable to provide the OpenCL library folder.")
self.library_dirs.append(cl_library_path)
if not hasattr(self, 'rpath'):
self.rpath = []
if sys.platform == "linux":
# Use relative RPATH to support out-of-source builds, i.e. pip install .
# Check https://man7.org/linux/man-pages/man8/ld.so.8.html for the $ORIGIN syntax
self.rpath.append("$ORIGIN/../pyarrow")
# Use absolute path so auditwheel and develop builds can find pyarrow.
self.rpath.extend(pa.get_library_dirs())
def build_extensions(self):
opts = ['-std=c++11', '-g']
if TILEDBVCF_DEBUG_BUILD:
opts.extend(['-O0'])
else:
opts.extend(['-O2'])
link_opts = []
for ext in self.extensions:
ext.extra_compile_args = opts
ext.extra_link_args = link_opts
import pyarrow
ext.include_dirs.append(pyarrow.get_include())
ext.libraries.extend(pyarrow.get_libraries())
# don't overlink the arrow core library
if 'arrow' in ext.libraries: ext.libraries.remove('arrow')
ext.library_dirs.extend(pyarrow.get_library_dirs())
find_or_build_libtiledbvcf(self)
build_ext.build_extensions(self)
os.path.join(CUDF_ROOT, "include"),
os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"),
os.path.join(
os.path.dirname(sysconfig.get_path("include")),
"libcudf/libcudacxx",
),
os.path.dirname(sysconfig.get_path("include")),
np.get_include(),
pa.get_include(),
cuda_include_dir,
],
library_dirs=(
pa.get_library_dirs()
+ [get_python_lib(), os.path.join(os.sys.prefix, "lib")]
),
libraries=["cudf"] + pa.get_libraries(),
language="c++",
extra_compile_args=["-std=c++14"],
)
]
setup(
name="cudf",
version=versioneer.get_version(),
description="cuDF - GPU Dataframe",
url="https://github.com/rapidsai/cudf",
author="NVIDIA Corporation",
license="Apache 2.0",
classifiers=[
"Intended Audience :: Developers",
"Topic :: Database",
from Cython.Build import cythonize
import os
import numpy as np
import pyarrow as pa
ext_modules = cythonize(
Extension("pyfletcher.lib", ["pyfletcher/lib.pyx"],
language="c++",
extra_compile_args=["-std=c++11", "-O3"],
extra_link_args=["-std=c++11"]))
for ext in ext_modules:
ext.include_dirs.append(np.get_include())
ext.include_dirs.append(pa.get_include())
ext.libraries.extend(pa.get_libraries())
ext.library_dirs.extend(pa.get_library_dirs())
ext.runtime_library_dirs.extend(pa.get_library_dirs())
ext.libraries.extend(["fletcher"])
ext.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0"))
this_directory = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
setup(name="pyfletcher",
version="0.0.5",
author="Lars van Leeuwen",
packages=['pyfletcher'],
description="A Python wrapper for the Fletcher runtime library",
long_description=long_description,
def build_extensions(self):
import pyarrow as pa
self.create_symlinks()
self.expand_sources()
self.copy_opencl_code()
# self.create_clang_tidy_compilation_db(self.extensions)
ct = self.compiler.compiler_type
c_opts, l_opts = self.create_options()
opts = c_opts.get(ct, [])
link_opts = l_opts.get(ct, [])
if sys.platform == "win32":
if "CL_INCLUDE_PATH" in os.environ:
cl_include_path = os.environ["CL_INCLUDE_PATH"]
else:
cl_include_path = find_opencl.find_opencl_include_dir()
if cl_include_path is None:
raise RuntimeError("OpenCL include path not found. Set \"CL_INCLUDE_PATH\" environment variable to provide the OpenCL headers folder.")
opts.append("/external:I" + cl_include_path)
# Include this because the name mangling affects to find the pyarrow functions.
opts.append("-D_GLIBCXX_USE_CXX11_ABI=0")
for ext in self.extensions:
ext.define_macros.append(("PYARROW_VERSION_INFO", pa.__version__))
# The compiled extension depends on a specific version of pyarrow.
self.distribution.install_requires = ['pybind11>=2.6', 'pyarrow=='+pa.__version__, "numpy"],
self.distribution.setup_requires = ['pybind11>=2.6', 'pyarrow=='+pa.__version__, "numpy"],
# opts.append("-g")
# opts.append("-O0")
# opts.append("-libstd=libc++")
# opts.append("-ferror-limit=1")
# opts.append("-Wno-unused-variable")
# opts.append("-Wno-unused-parameter")
# opts.append("-Wno-return-type")
# opts.append("-Wno-sign-compare")
# opts.append("-fsyntax-only")
# Activate debug mode.
# opts.append("-UNDEBUG")
# opts.append("-DDEBUG")
# This reduces the binary size because it removes the debug symbols. Check strip command to create release builds.
opts.append("-g0")
if ct == 'unix':
# opts.append("-march=native")
opts.append("-fdiagnostics-color=always")
opts.append("-Wall")
opts.append("-Wextra")
# opts.append(cpp_flag(self.compiler))
if has_flag(self.compiler, '-fvisibility=hidden'):
opts.append('-fvisibility=hidden')
for ext in self.extensions:
ext.extra_compile_args.extend(opts)
ext.extra_link_args.extend(link_opts)
# https://stackoverflow.com/questions/37752901/dylib-built-on-ci-cant-be-loaded
# Create the RPATH for MacOSX
if sys.platform == "darwin":
for ext in self.extensions:
ext.extra_link_args.append("-Wl,-rpath,@loader_path/../pyarrow")
ext.extra_link_args.append("-Wl,-rpath," + pa.get_library_dirs()[0])
build_ext.build_extensions(self)
# Copy the pyarrow dlls because Windows do not have the concept of RPATH.
if sys.platform == "win32":
for lib in pa.get_libraries():
import shutil
shutil.copyfile(pa.get_library_dirs()[0] + '/' + lib + '.dll',
path_to_build_folder() + '/' + lib + '.dll')
["@CMAKE_CURRENT_SOURCE_DIR@/Python/dbe.pyx"],
language="c++17",
include_dirs=[
np.get_include(),
pa.get_include(),
root,
"@CMAKE_SOURCE_DIR@",
"@CMAKE_CURRENT_SOURCE_DIR@",
"@CMAKE_SOURCE_DIR@/ThirdParty/rapidjson",
"@CMAKE_SOURCE_DIR@/Distributed/os",
],
library_dirs=pa.get_library_dirs() + ["@CMAKE_CURRENT_BINARY_DIR@", "."] +
extra_library_dirs,
runtime_library_dirs=pa.get_library_dirs() + ["$ORIGIN/../../"] +
extra_library_dirs,
libraries=pa.get_libraries() + ["DBEngine", "boost_system"],
extra_compile_args=["-std=c++17", "-DRAPIDJSON_HAS_STDSTRING"],
)
# Try uncommenting the following line on Linux
# if you get weird linker errors or runtime crashes
# dbe.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0"))
# "fat" wheel
data_files = []
if False: # TODO: implement an option?
data_files = [
("lib", ["$<TARGET_FILE:DBEngine>"]),
(
"bin",
[
"@CMAKE_BINARY_DIR@/bin/calcite-1.0-SNAPSHOT-jar-with-dependencies.jar"
from setuptools import setup, Extension
import os
import pyarrow as pa
import numpy as np
USE_CYTHON = False
ext_modules = [
Extension("avro_to_arrow._reader", ["src/avro_to_arrow/_reader.cpp"])
]
if USE_CYTHON:
from Cython.Build import cythonize
ext_modules = cythonize(["src/avro_to_arrow/_reader.pyx"],
language_level="3",
annotate=True)
for ext in ext_modules:
# The Numpy C headers are currently required
ext.include_dirs.append(np.get_include())
ext.include_dirs.append(pa.get_include())
if 'CONDA_DEFAULT_ENV' in os.environ.keys():
ext.include_dirs.append('/opt/mamba/envs/dev/include')
else:
ext.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0"))
ext.libraries.extend(['snappy'] + pa.get_libraries())
ext.library_dirs.extend(pa.get_library_dirs())
if os.name == 'posix':
ext.extra_compile_args.append('-std=c++11')
setup(ext_modules=ext_modules, zip_safe=False)
long_description=read('README.md'),
long_description_content_type='text/markdown',
url="https://github.com/abs-tudelft/fletcher",
project_urls={
"Bug Tracker": "https://github.com/abs-tudelft/fletcher/issues",
"Documentation": "https://abs-tudelft.github.io/fletcher/",
"Source Code": "https://github.com/abs-tudelft/fletcher/",
},
ext_modules=[
Extension(
"pyfletcher.lib", ["pyfletcher/lib.pyx"],
language="c++",
define_macros=[("_GLIBCXX_USE_CXX11_ABI", "0")],
include_dirs=[np.get_include(),
pa.get_include(), include_dir],
libraries=pa.get_libraries() + ["fletcher"],
library_dirs=pa.get_library_dirs() + lib_dirs,
runtime_library_dirs=pa.get_library_dirs() + lib_dirs,
extra_compile_args=["-std=c++11", "-O3"],
extra_link_args=["-std=c++11"])
],
install_requires=[
'numpy >= 1.14',
'pandas',
'pyarrow == 0.15.1',
],
setup_requires=[
'cython', 'numpy', 'pyarrow == 0.15.1', 'plumbum', 'pytest-runner'
],
tests_require=['pytest'],
classifiers=[
else:
os.environ['CXXFLAGS'] = 'std=c++11'
extra_link_args = [ "-v", # verbose
"-DSOME_DEFINE_OPT"]
extra_compile_args = [# "-v", # verbose # too much verbosity
# "-fopenmp",
"-O3",
"-w", # no warnings
# "-Wstrict-prototypes",
# "-Wimplicit-function-declaration",
# "-I/usr/local/include",
# "-I%s/samples/cli" % DB2PATH,
"-DSPCLIENT_PYTHON"]
libraries.extend(pyarrow.get_libraries())
library_dirs.extend(pyarrow.get_library_dirs())
if sys.version_info > (3,):
export_symbols = ['PyInit_spclient_python']
else:
export_symbols = ['initspclient_python']
def get_utilcli_c_location():
utilcli_c_location = os.path.join(provide_sample_cli_dir(), "utilcli.cpp")
return utilcli_c_location
class MyBuildExt(build_ext):
"""
library_dirs=lid + [
# for Linux
os.path.join(tbb_root, 'lib', 'intel64', 'gcc4.4'),
# for MacOS
os.path.join(tbb_root, 'lib'),
# for Windows
os.path.join(tbb_root, 'lib', 'intel64', 'vc_mt'),
],
language="c++")
ext_arrow_reader = Extension(
name="sdc.harrow_reader",
sources=["sdc/native/arrow_reader.cpp"],
extra_compile_args=eca,
extra_link_args=ela,
libraries=pa.get_libraries(),
include_dirs=["sdc/native/", numba_include_path,
pa.get_include()],
library_dirs=lid + pa.get_library_dirs(),
language="c++")
_ext_mods = [
ext_hdist,
ext_chiframes,
ext_set,
ext_str,
ext_dt,
ext_io,
ext_transport_seq,
ext_sort,
ext_conc_dict,
version="0.0.11",
author="Accelerated Big Data Systems, Delft University of Technology",
packages=find_packages(),
url="https://github.com/abs-tudelft/fletcher",
project_urls={
"Bug Tracker": "https://github.com/abs-tudelft/fletcher/issues",
"Documentation": "https://abs-tudelft.github.io/fletcher/",
"Source Code": "https://github.com/abs-tudelft/fletcher/",
},
ext_modules=[
Extension(
"pyfletchgen.lib", ["pyfletchgen/lib.pyx"],
language="c++",
include_dirs=[np.get_include(),
pa.get_include(), include_dir],
libraries=pa.get_libraries() + ["fletchgen_lib"],
library_dirs=pa.get_library_dirs() + lib_dirs,
runtime_library_dirs=pa.get_library_dirs() + lib_dirs,
extra_compile_args=["-std=c++11", "-O3"],
extra_link_args=["-std=c++11"])
],
entry_points={'console_scripts': ['fletchgen=pyfletchgen:_run']},
install_requires=[
'numpy >= 1.14',
'pandas',
'pyarrow == 1.0.0',
],
setup_requires=['cython', 'numpy', 'pyarrow == 1.0.0', 'plumbum'],
classifiers=[
"Programming Language :: Python :: 3",
"Programming Language :: Cython", "Intended Audience :: Developers",
"libcudf/libcudacxx",
),
os.path.dirname(sysconfig.get_path("include")),
np.get_include(),
pa.get_include(),
cuda_include_dir,
],
library_dirs=(
pa.get_library_dirs()
+ [
get_python_lib(),
os.path.join(os.sys.prefix, "lib"),
cuda_lib_dir,
]
),
libraries=["cudart", "cudf"] + pa.get_libraries() + ["arrow_cuda"],
language="c++",
extra_compile_args=["-std=c++14"],
)
]
setup(
name="cudf",
version=versioneer.get_version(),
description="cuDF - GPU Dataframe",
url="https://github.com/rapidsai/cudf",
author="NVIDIA Corporation",
license="Apache 2.0",
classifiers=[
"Intended Audience :: Developers",
"Topic :: Database",
