def test_cpu_vendor_linux(self):
"""Test getting CPU vendor (mocked for Linux)."""
st.get_os_type = lambda: st.LINUX
st.read_file = mocked_read_file
st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp)
global PROC_CPUINFO_TXT
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86
self.assertEqual(get_cpu_vendor(), INTEL)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
self.assertEqual(get_cpu_vendor(), IBM)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM
self.assertEqual(get_cpu_vendor(), ARM)
def _guess_aarch64_default_optarch(self):
"""
Guess default optarch for AARCH64 (vanilla ARM cores only)
This heuristic may fail if the CPU module is not supported by the GCC version being used.
"""
default_optarch = None
cpu_vendor = systemtools.get_cpu_vendor()
cpu_model = systemtools.get_cpu_model()
if cpu_vendor == systemtools.ARM and cpu_model.startswith('ARM '):
self.log.debug("Determining architecture-specific optimization flag for ARM (model: %s)", cpu_model)
core_types = []
for core_type in [ct.strip().lower() for ct in cpu_model[4:].split('+')]:
# Determine numeric ID for each core type, since we need to sort them later numerically
res = re.search('\d+$', core_type) # note: numeric ID is expected at the end
if res:
core_id = int(res.group(0))
core_types.append((core_id, core_type))
self.log.debug("Extracted numeric ID for ARM core type '%s': %s", core_type, core_id)
else:
# Bail out if we can't determine numeric ID
core_types = None
self.log.debug("Failed to extract numeric ID for ARM core type '%s', bailing out", core_type)
break
if core_types:
# On big.LITTLE setups, sort core types to have big core (higher model number) first.
# Example: 'mcpu=cortex-a72.cortex-a53' for "ARM Cortex-A53 + Cortex-A72"
default_optarch = 'mcpu=%s' % '.'.join([ct[1] for ct in sorted(core_types, reverse=True)])
self.log.debug("Using architecture-specific compiler optimization flag '%s'", default_optarch)
return default_optarch
def _guess_aarch64_default_optarch(self):
"""
Guess default optarch for AARCH64 (vanilla ARM cores only)
This heuristic may fail if the CPU module is not supported by the GCC version being used.
"""
default_optarch = None
cpu_vendor = systemtools.get_cpu_vendor()
cpu_model = systemtools.get_cpu_model()
if cpu_vendor == systemtools.ARM and cpu_model.startswith('ARM '):
self.log.debug("Determining architecture-specific optimization flag for ARM (model: %s)", cpu_model)
core_types = []
for core_type in [ct.strip().lower() for ct in cpu_model[4:].split('+')]:
# Determine numeric ID for each core type, since we need to sort them later numerically
res = re.search('\d+$', core_type) # note: numeric ID is expected at the end
if res:
core_id = int(res.group(0))
core_types.append((core_id, core_type))
self.log.debug("Extracted numeric ID for ARM core type '%s': %s", core_type, core_id)
else:
# Bail out if we can't determine numeric ID
core_types = None
self.log.debug("Failed to extract numeric ID for ARM core type '%s', bailing out", core_type)
break
if core_types:
# On big.LITTLE setups, sort core types to have big core (higher model number) first.
# Example: 'mcpu=cortex-a72.cortex-a53' for "ARM Cortex-A53 + Cortex-A72"
default_optarch = 'mcpu=%s' % '.'.join([ct[1] for ct in sorted(core_types, reverse=True)])
self.log.debug("Using architecture-specific compiler optimization flag '%s'", default_optarch)
return default_optarch
def _getOptimalArchitecture(self):
""" Get options for the current architecture """
optarchs = {systemtools.INTEL : 'xHOST', systemtools.AMD : 'msse3'}
if not self.arch:
self.arch = systemtools.get_cpu_vendor()
if self.arch in optarchs:
optarch = optarchs[self.arch]
self.log.info("Using %s as optarch for %s." % (optarch, self.arch))
return optarch
else:
self.log.error("Don't know how to set optarch for %s." % self.arch)
def test_cpu_vendor_linux(self):
"""Test getting CPU vendor (mocked for Linux)."""
st.get_os_type = lambda: st.LINUX
st.read_file = mocked_read_file
st.is_readable = lambda fp: mocked_is_readable(PROC_CPUINFO_FP, fp)
st.platform.uname = mocked_uname
global MACHINE_NAME
global PROC_CPUINFO_TXT
MACHINE_NAME = 'x86_64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_INTEL
self.assertEqual(get_cpu_vendor(), INTEL)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_AMD
self.assertEqual(get_cpu_vendor(), AMD)
MACHINE_NAME = 'ppc64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
self.assertEqual(get_cpu_vendor(), IBM)
MACHINE_NAME = 'armv7l'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_RASPI2
self.assertEqual(get_cpu_vendor(), ARM)
MACHINE_NAME = 'aarch64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_XGENE2
self.assertEqual(get_cpu_vendor(), APM)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_THUNDERX
self.assertEqual(get_cpu_vendor(), CAVIUM)
def test_cpu_vendor_linux(self):
"""Test getting CPU vendor (mocked for Linux)."""
st.get_os_type = lambda: st.LINUX
st.read_file = mocked_read_file
st.is_readable = lambda fp: mocked_is_readable(PROC_CPUINFO_FP, fp)
st.platform.uname = mocked_uname
global MACHINE_NAME
global PROC_CPUINFO_TXT
MACHINE_NAME = 'x86_64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_INTEL
self.assertEqual(get_cpu_vendor(), INTEL)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_AMD
self.assertEqual(get_cpu_vendor(), AMD)
MACHINE_NAME = 'ppc64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER
self.assertEqual(get_cpu_vendor(), IBM)
MACHINE_NAME = 'armv7l'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_RASPI2
self.assertEqual(get_cpu_vendor(), ARM)
MACHINE_NAME = 'aarch64'
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_XGENE2
self.assertEqual(get_cpu_vendor(), APM)
PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_THUNDERX
self.assertEqual(get_cpu_vendor(), CAVIUM)
def _get_optimal_architecture(self):
""" Get options for the current architecture """
if self.arch is None:
self.arch = systemtools.get_cpu_vendor()
if self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION is not None and \
self.arch in self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION:
optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[self.arch]
self.log.info("_get_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch))
self.options.options_map['optarch'] = optarch
if 'optarch' in self.options.options_map and self.options.options_map.get('optarch', None) is None:
self.log.raiseException("_get_optimal_architecture: don't know how to set optarch for %s." % self.arch)
def _get_optimal_architecture(self):
""" Get options for the current architecture """
if self.arch is None:
self.arch = systemtools.get_cpu_vendor()
if self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION is not None and \
self.arch in self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION:
optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[self.arch]
self.log.info(
"_get_optimal_architecture: using %s as optarch for %s." %
(optarch, self.arch))
self.options.options_map['optarch'] = optarch
if 'optarch' in self.options.options_map and self.options.options_map.get(
'optarch', None) is None:
self.log.raiseException(
"_get_optimal_architecture: don't know how to set optarch for %s."
% self.arch)
def test_cpu_vendor(self):
"""Test getting CPU vendor."""
cpu_vendor = get_cpu_vendor()
self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN])
def test_cpu_vendor_darwin(self):
"""Test getting CPU vendor (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
st.run_cmd = mocked_run_cmd
self.assertEqual(get_cpu_vendor(), INTEL)
def test_cpu_vendor_native(self):
"""Test getting CPU vendor."""
cpu_vendor = get_cpu_vendor()
self.assertTrue(cpu_vendor in CPU_VENDORS)
def configure_step(self):
# configure for 64-bit build
self.cfg.update('configopts', "-b 64")
if self.cfg['ignorethrottling']:
# ignore CPU throttling check
# this is not recommended, it will disturb the measurements done by ATLAS
# used for the EasyBuild demo, to avoid requiring root privileges
if LooseVersion(self.version) < LooseVersion('3.10.0'):
self.cfg.update('configopts', '-Si cputhrchk 0')
else:
self.log.warning(
"Ignore CPU throttling check is not possible via command line."
)
# apply patch to ignore CPU throttling: make ProbeCPUThrottle always return 0
# see http://sourceforge.net/p/math-atlas/support-requests/857/
cfg_file = os.path.join('CONFIG', 'src', 'config.c')
for line in fileinput.input(cfg_file,
inplace=1,
backup='.orig.eb'):
line = re.sub(r"^(\s*iret)\s*=\s*.*CPU THROTTLE.*$",
r"\1 = 0;", line)
sys.stdout.write(line)
self.log.warning('CPU throttling check ignored: NOT recommended!')
if get_cpu_vendor() in [AMD, INTEL]:
# use cycle accurate timer for timings
# see http://math-atlas.sourceforge.net/atlas_install/node23.html
# this should work on Linux with both GCC and Intel compilers
cpu_freq = int(get_cpu_speed())
self.cfg.update('configopts', "-D c -DPentiumCPS=%s" % cpu_freq)
else:
# use -DWALL for non-x86, see http://math-atlas.sourceforge.net/atlas_install/node25.html
self.cfg.update('configopts', "-D c -DWALL")
# if LAPACK is found, instruct ATLAS to provide a full LAPACK library
# ATLAS only provides a few LAPACK routines natively
if self.cfg['full_lapack']:
lapack_lib_version = LooseVersion('3.9')
if LooseVersion(self.version) < lapack_lib_version:
# pass built LAPACK library
lapack = get_software_root('LAPACK')
if lapack:
self.cfg.update(
'configopts',
' --with-netlib-lapack=%s/lib/liblapack.a' % lapack)
else:
raise EasyBuildError(
"netlib's LAPACK library not available, required to build ATLAS "
"with a full LAPACK library.")
else:
# pass LAPACK source tarball
lapack_src = None
for src in self.src:
if src['name'].startswith('lapack'):
lapack_src = src['path']
if lapack_src is not None:
self.cfg.update(
'configopts',
' --with-netlib-lapack-tarfile=%s' % lapack_src)
else:
raise EasyBuildError(
"LAPACK source tarball not available, but required.")
# enable building of shared libraries (requires -fPIC)
if self.cfg['sharedlibs'] or self.toolchain.options['pic']:
self.log.debug(
"Enabling -fPIC because we're building shared ATLAS libs, or just because."
)
self.cfg.update('configopts', '-Fa alg -fPIC')
# ATLAS only wants to be configured/built in a separate dir'
try:
objdir = "obj"
os.makedirs(objdir)
os.chdir(objdir)
except OSError as err:
raise EasyBuildError(
"Failed to create obj directory to build in: %s", err)
# specify compilers
self.cfg.update(
'configopts', '-C ic %(cc)s -C if %(f77)s' % {
'cc': os.getenv('CC'),
'f77': os.getenv('F77')
})
# call configure in parent dir
cmd = "%s %s/configure --prefix=%s %s" % (
self.cfg['preconfigopts'], self.cfg['start_dir'], self.installdir,
self.cfg['configopts'])
(out, exitcode) = run_cmd(cmd,
log_all=False,
log_ok=False,
simple=False)
if exitcode != 0:
throttling_regexp = re.compile("cpu throttling [a-zA-Z]* enabled",
re.IGNORECASE)
if throttling_regexp.search(out):
errormsg = (
"Configure failed, possible because CPU throttling is enabled; ATLAS doesn't like that. "
"You can either disable CPU throttling, or set 'ignorethrottling' to True in the ATLAS .eb file. "
"Also see http://math-atlas.sourceforge.net/errata.html#cputhrottle . "
"Configure output: %s") % out
else:
errormsg = "configure output: %s\nConfigure failed, not sure why (see output above)." % out
raise EasyBuildError(errormsg)
def test_cpu_vendor_darwin(self):
"""Test getting CPU vendor (mocked for Darwin)."""
st.get_os_type = lambda: st.DARWIN
st.run_cmd = mocked_run_cmd
self.assertEqual(get_cpu_vendor(), INTEL)
def configure_step(self):
# configure for 64-bit build
self.cfg.update('configopts', "-b 64")
if self.cfg['ignorethrottling']:
# ignore CPU throttling check
# this is not recommended, it will disturb the measurements done by ATLAS
# used for the EasyBuild demo, to avoid requiring root privileges
if LooseVersion(self.version) < LooseVersion('3.10.0'):
self.cfg.update('configopts', '-Si cputhrchk 0')
else:
self.log.warning("Ignore CPU throttling check is not possible via command line.")
# apply patch to ignore CPU throttling: make ProbeCPUThrottle always return 0
# see http://sourceforge.net/p/math-atlas/support-requests/857/
cfg_file = os.path.join('CONFIG', 'src', 'config.c')
for line in fileinput.input(cfg_file, inplace=1, backup='.orig.eb'):
line = re.sub(r"^(\s*iret)\s*=\s*.*CPU THROTTLE.*$", r"\1 = 0;", line)
sys.stdout.write(line)
self.log.warning('CPU throttling check ignored: NOT recommended!')
if get_cpu_vendor() in [AMD, INTEL]:
# use cycle accurate timer for timings
# see http://math-atlas.sourceforge.net/atlas_install/node23.html
# this should work on Linux with both GCC and Intel compilers
cpu_freq = int(get_cpu_speed())
self.cfg.update('configopts', "-D c -DPentiumCPS=%s" % cpu_freq)
else:
# use -DWALL for non-x86, see http://math-atlas.sourceforge.net/atlas_install/node25.html
self.cfg.update('configopts', "-D c -DWALL")
# if LAPACK is found, instruct ATLAS to provide a full LAPACK library
# ATLAS only provides a few LAPACK routines natively
if self.cfg['full_lapack']:
lapack_lib_version = LooseVersion('3.9')
if LooseVersion(self.version) < lapack_lib_version:
# pass built LAPACK library
lapack = get_software_root('LAPACK')
if lapack:
self.cfg.update('configopts', ' --with-netlib-lapack=%s/lib/liblapack.a' % lapack)
else:
raise EasyBuildError("netlib's LAPACK library not available, required to build ATLAS "
"with a full LAPACK library.")
else:
# pass LAPACK source tarball
lapack_src = None
for src in self.src:
if src['name'].startswith('lapack'):
lapack_src = src['path']
if lapack_src is not None:
self.cfg.update('configopts', ' --with-netlib-lapack-tarfile=%s' % lapack_src)
else:
raise EasyBuildError("LAPACK source tarball not available, but required.")
# enable building of shared libraries (requires -fPIC)
if self.cfg['sharedlibs'] or self.toolchain.options['pic']:
self.log.debug("Enabling -fPIC because we're building shared ATLAS libs, or just because.")
self.cfg.update('configopts', '-Fa alg -fPIC')
# ATLAS only wants to be configured/built in a separate dir'
try:
objdir = "obj"
os.makedirs(objdir)
os.chdir(objdir)
except OSError, err:
raise EasyBuildError("Failed to create obj directory to build in: %s", err)
def test_cpu_vendor(self):
"""Test getting CPU vendor."""
cpu_vendor = get_cpu_vendor()
self.assertTrue(cpu_vendor in VENDORS.values() + [UNKNOWN])
def configure_step(self):
# configure for 64-bit build
self.cfg.update('configopts', "-b 64")
if self.cfg['ignorethrottling']:
# ignore CPU throttling check
# this is not recommended, it will disturb the measurements done by ATLAS
# used for the EasyBuild demo, to avoid requiring root privileges
if LooseVersion(self.version) < LooseVersion('3.10.0'):
self.cfg.update('configopts', '-Si cputhrchk 0')
else:
self.log.warning("Ignore CPU throttling check is not possible via command line.")
# apply patch to ignore CPU throttling: make ProbeCPUThrottle always return 0
# see http://sourceforge.net/p/math-atlas/support-requests/857/
cfg_file = os.path.join('CONFIG', 'src', 'config.c')
for line in fileinput.input(cfg_file, inplace=1, backup='.orig.eb'):
line = re.sub(r"^(\s*iret)\s*=\s*.*CPU THROTTLE.*$", r"\1 = 0;", line)
sys.stdout.write(line)
self.log.warning('CPU throttling check ignored: NOT recommended!')
if get_cpu_vendor() in [AMD, INTEL]:
# use cycle accurate timer for timings
# see http://math-atlas.sourceforge.net/atlas_install/node23.html
# this should work on Linux with both GCC and Intel compilers
cpu_freq = int(get_cpu_speed())
self.cfg.update('configopts', "-D c -DPentiumCPS=%s" % cpu_freq)
else:
# use -DWALL for non-x86, see http://math-atlas.sourceforge.net/atlas_install/node25.html
self.cfg.update('configopts', "-D c -DWALL")
# if LAPACK is found, instruct ATLAS to provide a full LAPACK library
# ATLAS only provides a few LAPACK routines natively
if self.cfg['full_lapack']:
lapack_lib_version = LooseVersion('3.9')
if LooseVersion(self.version) < lapack_lib_version:
# pass built LAPACK library
lapack = get_software_root('LAPACK')
if lapack:
self.cfg.update('configopts', ' --with-netlib-lapack=%s/lib/liblapack.a' % lapack)
else:
raise EasyBuildError("netlib's LAPACK library not available, required to build ATLAS "
"with a full LAPACK library.")
else:
# pass LAPACK source tarball
lapack_src = None
for src in self.src:
if src['name'].startswith('lapack'):
lapack_src = src['path']
if lapack_src is not None:
self.cfg.update('configopts', ' --with-netlib-lapack-tarfile=%s' % lapack_src)
else:
raise EasyBuildError("LAPACK source tarball not available, but required.")
# enable building of shared libraries (requires -fPIC)
if self.cfg['sharedlibs'] or self.toolchain.options['pic']:
self.log.debug("Enabling -fPIC because we're building shared ATLAS libs, or just because.")
self.cfg.update('configopts', '-Fa alg -fPIC')
# ATLAS only wants to be configured/built in a separate dir'
try:
objdir = "obj"
os.makedirs(objdir)
os.chdir(objdir)
except OSError as err:
raise EasyBuildError("Failed to create obj directory to build in: %s", err)
# specify compilers
self.cfg.update('configopts', '-C ic %(cc)s -C if %(f77)s' % {
'cc':os.getenv('CC'),
'f77':os.getenv('F77')
})
# call configure in parent dir
cmd = "%s %s/configure --prefix=%s %s" % (self.cfg['preconfigopts'], self.cfg['start_dir'],
self.installdir, self.cfg['configopts'])
(out, exitcode) = run_cmd(cmd, log_all=False, log_ok=False, simple=False)
if exitcode != 0:
throttling_regexp = re.compile("cpu throttling [a-zA-Z]* enabled", re.IGNORECASE)
if throttling_regexp.search(out):
errormsg = ' '.join([
"Configure failed, possible because CPU throttling is enabled; ATLAS doesn't like that. ",
"You can either disable CPU throttling, or set 'ignorethrottling' to True in the ATLAS .eb spec file.",
"Also see http://math-atlas.sourceforge.net/errata.html#cputhrottle .",
"Configure output: %s",
]) % out
else:
errormsg = "configure output: %s\nConfigure failed, not sure why (see output above)." % out
raise EasyBuildError(errormsg)
def test_cpu_vendor_native(self):
"""Test getting CPU vendor."""
cpu_vendor = get_cpu_vendor()
self.assertTrue(cpu_vendor in CPU_VENDORS)
def configure_step(self):
# configure for 64-bit build
self.cfg.update('configopts', "-b 64")
if self.cfg['ignorethrottling']:
# ignore CPU throttling check
# this is not recommended, it will disturb the measurements done by ATLAS
# used for the EasyBuild demo, to avoid requiring root privileges
if LooseVersion(self.version) < LooseVersion('3.10.0'):
self.cfg.update('configopts', '-Si cputhrchk 0')
else:
self.log.warning(
"Ignore CPU throttling check is not possible via command line."
)
# apply patch to ignore CPU throttling: make ProbeCPUThrottle always return 0
# see http://sourceforge.net/p/math-atlas/support-requests/857/
cfg_file = os.path.join('CONFIG', 'src', 'config.c')
for line in fileinput.input(cfg_file,
inplace=1,
backup='.orig.eb'):
line = re.sub(r"^(\s*iret)\s*=\s*.*CPU THROTTLE.*$",
r"\1 = 0;", line)
sys.stdout.write(line)
self.log.warning('CPU throttling check ignored: NOT recommended!')
if get_cpu_vendor() in [AMD, INTEL]:
# use cycle accurate timer for timings
# see http://math-atlas.sourceforge.net/atlas_install/node23.html
# this should work on Linux with both GCC and Intel compilers
cpu_freq = int(get_cpu_speed())
self.cfg.update('configopts', "-D c -DPentiumCPS=%s" % cpu_freq)
else:
# use -DWALL for non-x86, see http://math-atlas.sourceforge.net/atlas_install/node25.html
self.cfg.update('configopts', "-D c -DWALL")
# if LAPACK is found, instruct ATLAS to provide a full LAPACK library
# ATLAS only provides a few LAPACK routines natively
if self.cfg['full_lapack']:
lapack_lib_version = LooseVersion('3.9')
if LooseVersion(self.version) < lapack_lib_version:
# pass built LAPACK library
lapack = get_software_root('LAPACK')
if lapack:
self.cfg.update(
'configopts',
' --with-netlib-lapack=%s/lib/liblapack.a' % lapack)
else:
raise EasyBuildError(
"netlib's LAPACK library not available, required to build ATLAS "
"with a full LAPACK library.")
else:
# pass LAPACK source tarball
lapack_src = None
for src in self.src:
if src['name'].startswith('lapack'):
lapack_src = src['path']
if lapack_src is not None:
self.cfg.update(
'configopts',
' --with-netlib-lapack-tarfile=%s' % lapack_src)
else:
raise EasyBuildError(
"LAPACK source tarball not available, but required.")
# enable building of shared libraries (requires -fPIC)
if self.cfg['sharedlibs'] or self.toolchain.options['pic']:
self.log.debug(
"Enabling -fPIC because we're building shared ATLAS libs, or just because."
)
self.cfg.update('configopts', '-Fa alg -fPIC')
# ATLAS only wants to be configured/built in a separate dir'
try:
objdir = "obj"
os.makedirs(objdir)
os.chdir(objdir)
except OSError, err:
raise EasyBuildError(
"Failed to create obj directory to build in: %s", err)
def test_cpu_vendor(self):
"""Test getting CPU vendor."""
cpu_vendor = get_cpu_vendor()
self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN])
