def generate_extensions(python_dirs):
extensions = []
for pdir in python_dirs:
mk = read_text_file(pjoin(dirname(abspath(__file__)), pdir,
"Makefile"))
makefile_dict = get_dict_from_buffer(mk, ['PROJECT', 'SOURCES'])
project = makefile_dict['PROJECT'][0].strip()
src_files = makefile_dict['SOURCES']
if project is None or src_files is None:
msg = "In directory = {0}, can not locate either the project "\
"name or the list of source files."\
"Got project = {1} and sources = {2}."\
.format(pdir, project, src_files)
raise AssertionError(msg)
sources = [pjoin(pdir, f) for f in src_files]
# print("Found project = {0} in dir = {1} with sources = {2}\n".
# format(project, pdir, sources))
ext = Extension("{0}".format(project), sources=sources)
extensions.append(ext)
return extensions
def setup_packages():
'''
Custom setup for Corrfunc package.
Optional: Set compiler via 'CC=/path/to/compiler' or
'CC /path/to/compiler' or 'CC = /path/to/compiler'
All the CC options are removed from sys.argv after
being parsed.
'''
# protect the user in case they run python setup.py not from root directory
src_path = dirname(abspath(sys.argv[0]))
old_path = os.getcwd()
os.chdir(src_path)
sys.path.insert(0, src_path)
# create a list of the python extensions
python_dirs = ["theory/python_bindings", "mocks/python_bindings"]
extensions = generate_extensions(python_dirs)
# check requirement for extensions and set the compiler if specified
# in command-line
common_dict = requirements_check()
# Some command options require headers/libs to be generated
# so that the following dirs_patters supplies them.
if install_required():
from distutils.sysconfig import get_config_var
if get_config_var('SHLIB_EXT') != '".so"' and version_info[0] == 2:
msg = "The extensions all get the `.so` automatically. "\
"However, python expects the extension to be `{0}`"\
.format(get_config_var('SHLIB_EXT'))
raise ValueError(msg)
# global variable compiler is set if passed in
# command-line
extra_string = ''
if compiler != '':
extra_string = 'CC={0}'.format(compiler)
command = "make libs {0}".format(extra_string)
run_command(command)
else:
# not installing. Check if creating source distribution
# in that case run distclean to delete auto-generated C
# files
if 'sdist' in sys.argv:
command = "make distclean"
run_command(command)
# find all the data-files required.
# Now the lib + associated header files have been generated
# and put in lib/ and include/
# This step must run after ``make install``
dirs_patterns = {
'theory/tests/data': ['*.ff', '*.txt', '*.txt.gz', '*.dat'],
'mocks/tests/data': ['*.ff', '*.txt', '*.txt.gz', '*.dat'],
'theory/tests': ['Mr19*', 'bins*', 'cmass*'],
'mocks/tests': ['Mr19*', 'bins*', 'angular_bins*'],
'include': ['count*.h'],
'lib': ['libcount*.a']
}
data_files = []
for d in dirs_patterns:
patterns = dirs_patterns[d]
f = recursive_glob(d, patterns)
data_files.extend(f)
# change them to be relative to package dir rather than root
data_files = ["../{0}".format(d) for d in data_files]
long_description = read_text_file('README.rst')
min_np_major = int(common_dict['MIN_NUMPY_MAJOR'][0])
min_np_minor = int(common_dict['MIN_NUMPY_MINOR'][0])
# All book-keeping is done.
# base_url = "https://github.com/manodeep/Corrfunc"
classifiers = [
'Development Status :: 4 - Beta', 'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Natural Language :: English', 'Operating System :: POSIX',
'Programming Language :: C', 'Programming Language :: Python',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6'
]
metadata = dict(
name=projectname,
version=version,
author='Manodeep Sinha',
author_email='*****@*****.**',
maintainer='Manodeep Sinha',
maintainer_email='*****@*****.**',
url=base_url,
download_url='{0}/archive/{1}-{2}.tar.gz'.format(
base_url, projectname, version),
description='Blazing fast correlation functions on the CPU',
long_description=long_description,
classifiers=classifiers,
license='MIT',
# Solaris might work, Windows will almost certainly not work
platforms=["Linux", "Mac OSX", "Unix"],
keywords=[
'correlation functions', 'simulations', 'surveys', 'galaxies'
],
provides=[projectname],
packages=find_packages(),
ext_package=projectname,
ext_modules=extensions,
package_data={'': data_files},
include_package_data=True,
setup_requires=[
'setuptools', 'numpy>={0}.{1}'.format(min_np_major, min_np_minor)
],
install_requires=[
'numpy>={0}.{1}'.format(min_np_major, min_np_minor), 'future',
'wurlitzer'
],
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4',
zip_safe=False,
cmdclass={'build_ext': BuildExtSubclass})
# Now the actual setup
try:
setup(**metadata)
finally:
del sys.path[0]
os.chdir(old_path)
return
def requirements_check():
common_mk_file = pjoin(dirname(abspath(__file__)), "common.mk")
common = read_text_file(common_mk_file)
common_dict = get_dict_from_buffer(common)
name = common_dict['DISTNAME'][0]
major = common_dict['MAJOR'][0]
minor = common_dict['MINOR'][0]
patch = common_dict['PATCHLEVEL'][0]
if name is None or major is None or minor is None or patch is None:
msg = "ERROR: Did not find at least one of the keys "\
"(DISTNAME, MAJOR, MINOR, PATCHLEVEL) in 'common.mk'.\n"\
"Checks can not run - aborting installation. "\
"projectname = {1} major = {2} minor = {3} patch = {4}\n\n"\
"You can fix this by re-issuing git clone {0}".\
format(base_url, projectname, major, minor, patch)
raise AssertionError(msg)
if projectname != name:
msg = 'Mis-match between C project name and python project name'\
'C claims project = {0} while python has {1}'.\
format(name, projectname)
raise AssertionError(msg)
global version
version = "{0}.{1}.{2}".format(major, minor, patch)
# Check that version matches
if Corrfunc.__version__ != version:
msg = "ERROR: Version mis-match. Python version found = {0} \
while C version claims {1}".format(Corrfunc.__version__, version)
raise AssertionError(msg)
# Okay common.mk has been updated to use current python
# for building the extensions as required. Now check for
# min. python version
min_py_major = int(common_dict['MIN_PYTHON_MAJOR'][0])
min_py_minor = int(common_dict['MIN_PYTHON_MINOR'][0])
# Enforce minimum python version
if version_info[0] < min_py_major or \
(version_info[0] == min_py_major and version_info[1] < min_py_minor):
msg = "Sorry. Found python {0}.{1} but minimum required "\
"python version is {2}.{3}".format(version_info[0],
version_info[1],
min_py_major,
min_py_minor)
raise AssertionError(msg)
# Since arbitrary python can be used even within the Makefile
# make sure that the current python executable is the same as the
# one specified in common.mk. Easiest way is to replace,
# but no need to do so if creating a source distribution
if 'sdist' not in sys.argv:
this_python = sys.executable
key = "PYTHON"
replacement = '\n{0}:={1}'.format(key, this_python)
common = replace_first_key_in_makefile(common, key, replacement,
common_mk_file)
# Check if CC is in argv:
CC = "CC"
for iarg, arg in enumerate(sys.argv):
if CC not in arg:
continue
if '=' in arg:
# user passed `CC=/path/to/compiler`
# therefore, split on '=' to get the
# compiler (i.e, 'CC') and the value
# (i.e, the name/path of compiler)
key, value = arg.strip().split('=')
else:
# Space-separated or spaces and an '=' sign
key = arg.strip()
if key != CC:
msg = "Something strange has happened. Expected to find "\
"a custom compiler from the command-line but \n"\
"found command-line argument '{0}' (that matches "\
"pattern ['CC=/path/to/compiler']). Parsing "\
"produced CC={1}".format(arg, key)
raise ValueError(msg)
check_arg = iarg + 1
# Is there an "=" sign (i.e., `CC = /path/to/compiler`)
# or did the user simply pass `CC /path/to/compiler`
if check_arg >= len(sys.argv):
msg = "Found compiler key = {} but could not locate "\
"compiler value - no further command-line "\
"parameters were passed.\nPlease pass the "\
"custom compiler name either `CC=compiler`"\
"or as `CC=/path/to/compiler`".format(key)
raise ValueError(msg)
# The user could have specified `CC =compiler` or
# `CC = compiler`. The following 'if' condition checks
# for the first case, the 'else' checks for the second
# case (`CC = compiler`)
if '=' in sys.argv[check_arg] and \
sys.argv[check_arg].strip() != '=':
_, value = sys.argv[check_arg].strip().split('=')
else:
# Otherwise, there was white-space separated '='
# we can delete that command-line argument containing
# just the '=' sign.
del sys.argv[check_arg]
# should be parsing the compiler value now
if check_arg >= len(sys.argv):
msg = "Found compiler key = CC but could not locate "\
"compiler value (either as `CC=/path/to/CC` "\
"or as `CC /path/to/CC`"
raise ValueError(msg)
value = sys.argv[check_arg].strip()
# this deletes the argument containing the compiler name
del sys.argv[check_arg]
if key != CC or value == '':
msg = "Something strange has happened. Expected to find a "\
"custom compiler from the command-line but found \n"\
"command-line argument '{0}' (that matches pattern "\
"['CC=/path/to/compiler']). Parsing produced CC={1} "\
"and $CC={2}".format(arg, key, value)
raise ValueError(msg)
# check if value is a valid compiler
full_compiler = which(value)
if full_compiler is None:
msg = "Found compiler = '{0}' on the command-line but '{0}' "\
"can not be resolved from the shell.\n"\
"Please specify CC=/path/to/compiler in the "\
"python -m pip setup.py call.".format(value)
raise ValueError(msg)
replacement = '\n{0}:={1}'.format(CC, value)
replace_first_key_in_makefile(common, CC, replacement, common_mk_file)
global compiler
compiler = value
# Delete the 'CC' key, the compiler name and the '='
# have already been deleted
del sys.argv[iarg]
break
return common_dict
def requirements_check():
common_mk_file = pjoin(dirname(abspath(__file__)), "common.mk")
common = read_text_file(common_mk_file)
common_dict = get_dict_from_buffer(common)
name = common_dict['DISTNAME'][0]
major = common_dict['MAJOR'][0]
minor = common_dict['MINOR'][0]
patch = common_dict['PATCHLEVEL'][0]
if name is None or major is None or minor is None or patch is None:
msg = "ERROR: Did not find at least one of the keys "\
"(DISTNAME, MAJOR, MINOR, PATCHLEVEL) in 'common.mk'.\n"\
"Checks can not run - aborting installation. "\
"projectname = {1} major = {2} minor = {3} patch = {4}\n\n"\
"You can fix this by re-issuing git clone {0}".\
format(base_url, projectname, major, minor, patch)
raise AssertionError(msg)
if projectname != name:
msg = 'Mis-match between C project name and python project name'\
'C claims project = {0} while python has {1}'.\
format(name, projectname)
raise AssertionError(msg)
global version
version = "{0}.{1}.{2}".format(major, minor, patch)
# Check that version matches
if Corrfunc.__version__ != version:
msg = "ERROR: Version mis-match. Python version found = {0} \
while C version claims {1}".format(Corrfunc.__version__, version)
raise AssertionError(msg)
# Since arbitrary python can be used even within the Makefile
# make sure that the current python executable is the same as the
# one specified in common.mk. Easiest way is to replace
make_python = common_dict['PYTHON'][0]
if make_python is None:
msg = "PYTHON is not defined in 'common.mk'. Please "\
"edit 'common.mk' and define PYTHON (typically "\
"just python) "
raise AssertionError(msg)
this_python = sys.executable
python_script = "'from __future__ import print_function; "\
"import sys; print(sys.executable)'"
get_full_python, full_python_errors = run_command(make_python + " -c " +
python_script,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if get_full_python is None:
msg = "Could not determine which python is resolved in the Makefile "\
"Parsed PYTHON=[${0}] in Makefile which could not be resolved "\
"through the shell. Please report your python setup and file "\
"an installation issue at {1}.".format(make_python, base_url)
raise RuntimeError(msg)
get_full_python = strip_line(get_full_python, os.linesep)
if get_full_python != this_python:
msg = "Looks like python specified in Makefile = {0} is different "\
"from the invoked python instance = {1}.\nReplacing PYTHON "\
"in 'common.mk' and recompiling *all* files".format(
get_full_python, this_python)
print(msg)
key = "PYTHON"
replacement = '\n{0}:={1}'.format(key, this_python)
common = replace_first_key_in_makefile(common, key, replacement,
common_mk_file)
# Okay common.mk has now been updated to use current python
# for building the extensions as required
min_py_major = int(common_dict['MIN_PYTHON_MAJOR'][0])
min_py_minor = int(common_dict['MIN_PYTHON_MINOR'][0])
# Enforce minimum python version
if version_info[0] < min_py_major or \
(version_info[0] == min_py_major and version_info[1] < min_py_minor):
msg = "Sorry. Found python {0}.{1} but minimum required "\
"python version is {2}.{3}".format(version_info[0],
version_info[1],
min_py_major,
min_py_minor)
raise AssertionError(msg)
# Check if CC is in argv:
CC = "CC"
for iarg, arg in enumerate(sys.argv):
if CC in arg:
if '=' in arg:
key, value = arg.strip().split('=')
else:
# Space-separated or no spaces and equal
key = arg.strip()
check_arg = iarg + 1
if key != CC:
msg = "Something strange has happened. Expected to find "\
"a custom compiler from the command-line but \n"\
"found command-line argument '{0}' (that matches "\
"pattern ['CC=/path/to/compiler']). Parsing "\
"produced CC={1}".format(arg, key)
raise ValueError(msg)
# Is there an "=" sign or did the user
# simply pass `CC /path/to/compiler`
if check_arg < len(sys.argv):
if sys.argv[check_arg] == '=':
# skip '=' sign
del sys.argv[check_arg]
# should be parsing the compiler value now
if not check_arg < len(sys.argv):
msg = "Found compiler key = CC but could not locate "\
"compiler value (either as `CC=/path/to/CC` "\
"or as `CC /path/to/CC`"
raise ValueError(msg)
value = sys.argv[check_arg].strip()
del sys.argv[check_arg]
if key != CC or value == '':
msg = "Something strange has happened. Expected to find a "\
"custom compiler from the command-line but found \n"\
"command-line argument '{0}' (that matches pattern "\
"['CC=/path/to/compiler']). Parsing produced CC={1} "\
"and $CC={2}".format(arg, key, value)
raise ValueError(msg)
# check if value is a valid compiler
full_compiler = which(value)
if full_compiler is None:
msg = "Found compiler = '{0}' on the command-line but '{0}' "\
"can not be resolved from the shell.\n"\
"Please specify CC=/path/to/compiler in the "\
"python setup.py call.".format(value)
raise ValueError(msg)
replacement = '\n{0}:={1}'.format(CC, value)
replace_first_key_in_makefile(common, CC, replacement,
common_mk_file)
del sys.argv[iarg]
global compiler
compiler = value
break
return common_dict