/
setup_configure.py
296 lines (241 loc) · 10.3 KB
/
setup_configure.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
"""
Implements a new custom Distutils command for handling library
configuration.
The "configure" command here doesn't directly affect things like
config.pxi; rather, it exists to provide a set of attributes that are
used by the build_ext replacement in setup_build.py.
Options from the command line and environment variables are stored
between invocations in a pickle file. This allows configuring the library
once and e.g. calling "build" and "test" without recompiling everything
or explicitly providing the same options every time.
This module also contains the auto-detection logic for figuring out
the currently installed HDF5 version.
"""
from distutils.cmd import Command
import os
import os.path as op
import sys
import pickle
def loadpickle():
""" Load settings dict from the pickle file """
try:
with open('h5config.pkl','rb') as f:
cfg = pickle.load(f)
if not isinstance(cfg, dict): raise TypeError
except Exception:
return {}
return cfg
def savepickle(dct):
""" Save settings dict to the pickle file """
with open('h5config.pkl','wb') as f:
pickle.dump(dct, f, protocol=0)
def validate_version(s):
""" Ensure that s contains an X.Y.Z format version string, or ValueError.
"""
try:
tpl = tuple(int(x) for x in s.split('.'))
if len(tpl) != 3: raise ValueError
except Exception:
raise ValueError("HDF5 version string must be in X.Y.Z format")
def get_env_options():
# The keys here match the option attributes on *configure*
return {
'hdf5': os.environ.get('HDF5_DIR'),
'hdf5_includedir': os.environ.get('HDF5_INCLUDEDIR'),
'hdf5_libdir': os.environ.get('HDF5_LIBDIR'),
'hdf5_pkgconfig_name': os.environ.get('HDF5_PKGCONFIG_NAME'),
'hdf5_version': os.environ.get('HDF5_VERSION'),
'mpi': os.environ.get('HDF5_MPI') == "ON",
}
class configure(Command):
"""
Configure build options for h5py: custom path to HDF5, version of
the HDF5 library, and whether MPI is enabled.
Options can come from either command line options or environment
variables (but specifying the same option in both is an error).
Options not specified will be loaded from the previous configuration,
so they are 'sticky' (except hdf5-version).
When options change, the rebuild_required attribute is set, and
may only be reset by calling reset_rebuild(). The custom build_ext
command does this.
"""
description = "Configure h5py build options"
user_options = [('hdf5=', 'h', 'Custom path prefix to HDF5'),
('hdf5-version=', '5', 'HDF5 version "X.Y.Z"'),
('hdf5-includedir=', 'i', 'path to HDF5 headers'),
('hdf5-libdir=', 'l', 'path to HDF5 library'),
('hdf5-pkgconfig-name=', 'p', 'name of HDF5 pkgconfig file'),
('mpi', 'm', 'Enable MPI building'),
('reset', 'r', 'Reset config options') ]
def initialize_options(self):
self.hdf5 = None
self.hdf5_version = None
self.hdf5_includedir = None
self.hdf5_libdir = None
self.hdf5_pkgconfig_name = None
self.mpi = None
self.reset = None
def finalize_options(self):
# Merge environment options with command-line
for setting, env_val in get_env_options().items():
if env_val is not None:
if getattr(self, setting) is not None:
raise ValueError(
f"Provide {setting} in command line or environment "
f"variable, not both."
)
setattr(self, setting, env_val)
if sum([
bool(self.hdf5_includedir or self.hdf5_libdir),
bool(self.hdf5),
bool(self.hdf5_pkgconfig_name)
]) > 1:
raise ValueError(
"Specify only one of: HDF5 lib/include dirs, HDF5 prefix dir, "
"or HDF5 pkgconfig name"
)
# Check version number format
if self.hdf5_version is not None:
validate_version(self.hdf5_version)
def reset_rebuild(self):
""" Mark this configuration as built """
dct = loadpickle()
dct['rebuild'] = False
savepickle(dct)
def _find_hdf5_compiler_settings(self, olds, mpi):
"""Returns (include_dirs, lib_dirs, define_macros)"""
# Specified lib/include dirs explicitly
if self.hdf5_includedir or self.hdf5_libdir:
inc_dirs = [self.hdf5_includedir] if self.hdf5_includedir else []
lib_dirs = [self.hdf5_libdir] if self.hdf5_libdir else []
return (inc_dirs, lib_dirs, [])
# Specified a prefix dir (e.g. '/usr/local')
if self.hdf5:
inc_dirs = [op.join(self.hdf5, 'include')]
lib_dirs = [op.join(self.hdf5, 'lib')]
if sys.platform.startswith('win'):
lib_dirs.append(op.join(self.hdf5, 'bin'))
return (inc_dirs, lib_dirs, [])
# Specified a name to be looked up in pkgconfig
if self.hdf5_pkgconfig_name:
import pkgconfig
if not pkgconfig.exists(self.hdf5_pkgconfig_name):
raise ValueError(
f"No pkgconfig information for {self.hdf5_pkgconfig_name}"
)
pc = pkgconfig.parse(self.hdf5_pkgconfig_name)
return (pc['include_dirs'], pc['library_dirs'], pc['define_macros'])
# Re-use previously specified settings
if olds.get('hdf5_includedirs') and olds.get('hdf5_libdirs'):
return (
olds['hdf5_includedirs'],
olds['hdf5_libdirs'],
olds.get('hdf5_define_macros', []),
)
# Fallback: query pkgconfig for default hdf5 names
import pkgconfig
pc_name = 'hdf5-openmpi' if mpi else 'hdf5'
pc = {}
try:
if pkgconfig.exists(pc_name):
pc = pkgconfig.parse(pc_name)
except EnvironmentError:
if os.name != 'nt':
print(
"Building h5py requires pkg-config unless the HDF5 path "
"is explicitly specified", file=sys.stderr
)
raise
return (
pc.get('include_dirs', []),
pc.get('library_dirs', []),
pc.get('define_macros', []),
)
def run(self):
""" Distutils calls this when the command is run """
# Step 1: Load previous settings and combine with current ones
oldsettings = {} if self.reset else loadpickle()
if self.mpi is None:
self.mpi = oldsettings.get('mpi', False)
self.hdf5_includedirs, self.hdf5_libdirs, self.hdf5_define_macros = \
self._find_hdf5_compiler_settings(oldsettings, self.mpi)
# Don't use the HDF5 version saved previously - that may be referring
# to another library. It should always be specified or autodetected.
# The HDF5 version is persisted only so we can check if it changed.
if self.hdf5_version is None:
self.hdf5_version = autodetect_version(self.hdf5_libdirs)
# Step 2: determine if a rebuild is needed & save the settings
current_settings = {
'hdf5_includedirs': self.hdf5_includedirs,
'hdf5_libdirs': self.hdf5_libdirs,
'hdf5_define_macros': self.hdf5_define_macros,
'hdf5_version': self.hdf5_version,
'mpi': self.mpi,
'rebuild': False,
}
self.rebuild_required = current_settings['rebuild'] = (
# If we haven't built since a previous config change
oldsettings.get('rebuild')
# If the config has changed now
or current_settings != oldsettings
# Corner case: If options reset, but only if they previously
# had non-default values (to handle multiple resets in a row)
or bool(self.reset and any(loadpickle().values()))
)
savepickle(current_settings)
# Step 3: print the resulting configuration to stdout
def fmt_dirs(l):
return '\n'.join((['['] + [f' {d!r}' for d in l] + [']'])) if l else '[]'
print('*' * 80)
print(' ' * 23 + "Summary of the h5py configuration")
print('')
print("HDF5 include dirs:", fmt_dirs(self.hdf5_includedirs))
print("HDF5 library dirs:", fmt_dirs(self.hdf5_libdirs))
print(" HDF5 Version:", repr(self.hdf5_version))
print(" MPI Enabled:", self.mpi)
print(" Rebuild Required:", self.rebuild_required)
print('')
print('*' * 80)
def autodetect_version(libdirs):
"""
Detect the current version of HDF5, and return X.Y.Z version string.
Intended for Unix-ish platforms (Linux, OS X, BSD).
Does not support Windows. Raises an exception if anything goes wrong.
config: configuration for the build (configure command)
"""
import re
import ctypes
from ctypes import byref
if sys.platform.startswith('darwin'):
default_path = 'libhdf5.dylib'
regexp = re.compile(r'^libhdf5.dylib')
elif sys.platform.startswith('win') or \
sys.platform.startswith('cygwin'):
default_path = 'hdf5.dll'
regexp = re.compile(r'^hdf5.dll')
else:
default_path = 'libhdf5.so'
regexp = re.compile(r'^libhdf5.so')
path = None
for d in libdirs:
try:
candidates = [x for x in os.listdir(d) if regexp.match(x)]
except Exception:
continue # Skip invalid entries
if len(candidates) != 0:
candidates.sort(key=lambda x: len(x)) # Prefer libfoo.so to libfoo.so.X.Y.Z
path = op.abspath(op.join(d, candidates[0]))
break
if path is None:
path = default_path
major = ctypes.c_uint()
minor = ctypes.c_uint()
release = ctypes.c_uint()
print("Loading library to get version:", path)
try:
lib = ctypes.cdll.LoadLibrary(path)
lib.H5get_libversion(byref(major), byref(minor), byref(release))
except:
print("error: Unable to load dependency HDF5, make sure HDF5 is installed properly")
raise
return "{0}.{1}.{2}".format(int(major.value), int(minor.value), int(release.value))