def test_is_readable(self):
"""Test is_readable"""
test_file = os.path.join(self.test_prefix, 'test.txt')
self.assertFalse(ft.is_readable(test_file))
ft.write_file(test_file, 'test')
self.assertTrue(ft.is_readable(test_file))
os.chmod(test_file, 0)
self.assertFalse(ft.is_readable(test_file))
def get_cpu_speed():
"""
Returns the (maximum) cpu speed in MHz, as a float value.
In case of throttling, the highest cpu speed is returns.
"""
cpu_freq = None
os_type = get_os_type()
if os_type == LINUX:
# Linux with cpu scaling
if is_readable(MAX_FREQ_FP):
_log.debug("Trying to determine CPU frequency on Linux via %s" %
MAX_FREQ_FP)
txt = read_file(MAX_FREQ_FP)
cpu_freq = float(txt) // 1000
# Linux without cpu scaling
elif is_readable(PROC_CPUINFO_FP):
_log.debug("Trying to determine CPU frequency on Linux via %s" %
PROC_CPUINFO_FP)
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
# 'cpu MHz' on Linux/x86 (& more), 'clock' on Linux/POWER
cpu_freq_regex = re.compile(
r"^(?:cpu MHz|clock)\s*:\s*(?P<cpu_freq>\d+(?:\.\d+)?)", re.M)
res = cpu_freq_regex.search(proc_cpuinfo)
if res:
cpu_freq = float(res.group('cpu_freq'))
_log.debug("Found CPU frequency using regex '%s': %s" %
(cpu_freq_regex.pattern, cpu_freq))
else:
_log.debug("Failed to determine CPU frequency from %s",
PROC_CPUINFO_FP)
else:
_log.debug(
"%s not found to determine max. CPU clock frequency without CPU scaling",
PROC_CPUINFO_FP)
elif os_type == DARWIN:
cmd = "sysctl -n hw.cpufrequency_max"
_log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" %
cmd)
out, ec = run_cmd(cmd,
force_in_dry_run=True,
trace=False,
stream_output=False)
if ec == 0:
# returns clock frequency in cycles/sec, but we want MHz
cpu_freq = float(out.strip()) // (1000**2)
else:
raise SystemToolsException(
"Could not determine CPU clock frequency (OS: %s)." % os_type)
return cpu_freq
def test_is_readable(self):
"""Test is_readable"""
test_file = os.path.join(self.test_prefix, 'test.txt')
self.assertFalse(ft.is_readable(test_file))
ft.write_file(test_file, 'test')
self.assertTrue(ft.is_readable(test_file))
os.chmod(test_file, 0)
self.assertFalse(ft.is_readable(test_file))
def get_cpu_family():
"""
Determine CPU family.
:return: a value from the CPU_FAMILIES list
"""
family = None
vendor = get_cpu_vendor()
if vendor in CPU_FAMILIES:
family = vendor
_log.debug("Using vendor as CPU family: %s" % family)
else:
arch = get_cpu_architecture()
if arch in [AARCH32, AARCH64]:
# Custom ARM-based designs from other vendors
family = ARM
# POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo
elif is_readable(PROC_CPUINFO_FP):
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M)
if power_regex.search(proc_cpuinfo):
family = POWER
_log.debug(
"Determined CPU family using regex '%s' in %s: %s", power_regex.pattern, PROC_CPUINFO_FP, family
)
if family is None:
family = UNKNOWN
_log.warning("Failed to determine CPU family, returning %s" % family)
return family
def install_step(self):
"""Custom install procedure for TensorFlow."""
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if len(whl_paths) == 1:
# --ignore-installed is required to ensure *this* wheel is installed
cmd = "pip install --ignore-installed --prefix=%s %s" % (self.installdir, whl_paths[0])
# if extensions are listed, assume they will provide all required dependencies,
# so use --no-deps to prevent pip from downloading & installing them
if self.cfg['exts_list']:
cmd += ' --no-deps'
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s", whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341
# If the site-packages/google/__init__.py file is missing, make
# it an empty file.
# This fixes the "No module named google.protobuf" error that
# sometimes shows up during sanity_check
google_protobuf_dir = os.path.join(self.installdir, self.pylibdir, 'google', 'protobuf')
google_init_file = os.path.join(self.installdir, self.pylibdir, 'google', '__init__.py')
if os.path.isdir(google_protobuf_dir) and not is_readable(google_init_file):
self.log.debug("Creating (empty) missing %s", google_init_file)
write_file(google_init_file, '')
def get_total_memory():
"""
Try to ascertain this node's total memory
:return: total memory as an integer, specifically a number of megabytes
"""
memtotal = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_MEMINFO_FP):
_log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP)
meminfo = read_file(PROC_MEMINFO_FP)
mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M)
if mem_mo:
memtotal = int(mem_mo.group(1)) // 1024
elif os_type == DARWIN:
cmd = "sysctl -n hw.memsize"
_log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
memtotal = int(out.strip()) // (1024**2)
if memtotal is None:
memtotal = UNKNOWN
_log.warning("Failed to determine total memory, returning %s", memtotal)
return memtotal
def install_step(self):
"""Custom install procedure for TensorFlow."""
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if len(whl_paths) == 1:
# --ignore-installed is required to ensure *this* wheel is installed
cmd = "pip install --ignore-installed --prefix=%s %s" % (self.installdir, whl_paths[0])
# if extensions are listed, assume they will provide all required dependencies,
# so use --no-deps to prevent pip from downloading & installing them
if self.cfg['exts_list']:
cmd += ' --no-deps'
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s", whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341
# If the site-packages/google/__init__.py file is missing, make
# it an empty file.
# This fixes the "No module named google.protobuf" error that
# sometimes shows up during sanity_check
google_protobuf_dir = os.path.join(self.installdir, self.pylibdir, 'google', 'protobuf')
google_init_file = os.path.join(self.installdir, self.pylibdir, 'google', '__init__.py')
if os.path.isdir(google_protobuf_dir) and not is_readable(google_init_file):
self.log.debug("Creating (empty) missing %s", google_init_file)
write_file(google_init_file, '')
def get_cpu_family():
"""
Determine CPU family.
:return: a value from the CPU_FAMILIES list
"""
family = None
vendor = get_cpu_vendor()
if vendor in CPU_FAMILIES:
family = vendor
_log.debug("Using vendor as CPU family: %s" % family)
else:
arch = get_cpu_architecture()
if arch in [AARCH32, AARCH64]:
# Custom ARM-based designs from other vendors
family = ARM
# POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo
elif is_readable(PROC_CPUINFO_FP):
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M)
if power_regex.search(proc_cpuinfo):
family = POWER
_log.debug("Determined CPU family using regex '%s' in %s: %s",
power_regex.pattern, PROC_CPUINFO_FP, family)
if family is None:
family = UNKNOWN
_log.warning("Failed to determine CPU family, returning %s" % family)
return family
def get_total_memory():
"""
Try to ascertain this node's total memory
:return: total memory as an integer, specifically a number of megabytes
"""
memtotal = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_MEMINFO_FP):
_log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP)
meminfo = read_file(PROC_MEMINFO_FP)
mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M)
if mem_mo:
memtotal = int(mem_mo.group(1)) / 1024
elif os_type == DARWIN:
cmd = "sysctl -n hw.memsize"
_log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
memtotal = int(out.strip()) / (1024**2)
if memtotal is None:
memtotal = UNKNOWN
_log.warning("Failed to determine total memory, returning %s", memtotal)
return memtotal
def get_cpu_features():
"""
Get list of CPU features
"""
cpu_feat = []
os_type = get_os_type()
if os_type == LINUX:
if is_readable(PROC_CPUINFO_FP):
_log.debug("Trying to determine CPU features on Linux via %s",
PROC_CPUINFO_FP)
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
# 'flags' on Linux/x86, 'Features' on Linux/ARM
flags_regex = re.compile(
r"^(?:flags|[fF]eatures)\s*:\s*(?P<flags>.*)", re.M)
res = flags_regex.search(proc_cpuinfo)
if res:
cpu_feat = sorted(res.group('flags').lower().split())
_log.debug("Found CPU features using regex '%s': %s",
flags_regex.pattern, cpu_feat)
elif get_cpu_architecture() == POWER:
# for Linux@POWER systems, no flags/features are listed, but we can check for Altivec
cpu_altivec_regex = re.compile("^cpu\s*:.*altivec supported",
re.M)
if cpu_altivec_regex.search(proc_cpuinfo):
cpu_feat.append('altivec')
# VSX is supported since POWER7
cpu_power7_regex = re.compile("^cpu\s*:.*POWER(7|8|9)", re.M)
if cpu_power7_regex.search(proc_cpuinfo):
cpu_feat.append('vsx')
else:
_log.debug("Failed to determine CPU features from %s",
PROC_CPUINFO_FP)
else:
_log.debug("%s not found to determine CPU features",
PROC_CPUINFO_FP)
elif os_type == DARWIN:
for feature_set in ['extfeatures', 'features', 'leaf7_features']:
cmd = "sysctl -n machdep.cpu.%s" % feature_set
_log.debug(
"Trying to determine CPU features on Darwin via cmd '%s'", cmd)
out, ec = run_cmd(cmd,
force_in_dry_run=True,
trace=False,
stream_output=False)
if ec == 0:
cpu_feat.extend(out.strip().lower().split())
cpu_feat.sort()
else:
raise SystemToolsException(
"Could not determine CPU features (OS: %s)" % os_type)
return cpu_feat
def get_cpu_model():
"""
Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
"""
model = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_CPUINFO_FP):
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
arch = get_cpu_architecture()
if arch in [AARCH32, AARCH64]:
# On ARM platforms, no model name is provided in /proc/cpuinfo. However, for vanilla ARM cores
# we can reverse-map the part number.
vendor = get_cpu_vendor()
if vendor == ARM:
model_regex = re.compile(r"CPU part\s+:\s*(\S+)", re.M)
# There can be big.LITTLE setups with different types of cores!
model_ids = model_regex.findall(proc_cpuinfo)
if model_ids:
id_list = []
for model_id in sorted(set(model_ids)):
id_list.append(ARM_CORTEX_IDS.get(model_id, UNKNOWN))
model = vendor + " " + " + ".join(id_list)
_log.debug(
"Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern,
PROC_CPUINFO_FP,
model,
)
else:
# we need 'model name' on Linux/x86, but 'model' is there first with different info
# 'model name' is not there for Linux/POWER, but 'model' has the right info
model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M)
res = model_regex.search(proc_cpuinfo)
if res is not None:
model = res.group("model").strip()
_log.debug(
"Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern,
PROC_CPUINFO_FP,
model,
)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
out, ec = run_cmd(cmd, force_in_dry_run=True)
if ec == 0:
model = out.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
if model is None:
model = UNKNOWN
_log.warning("Failed to determine CPU model, returning %s" % model)
return model
def install_step(self):
"""Custom install procedure for TensorFlow."""
# avoid that pip (ab)uses $HOME/.cache/pip
# cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
env.setvar('XDG_CACHE_HOME', tempfile.gettempdir())
self.log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME'])
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if not whl_paths:
whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow-*.whl'))
if len(whl_paths) == 1:
# --ignore-installed is required to ensure *this* wheel is installed
cmd = "pip install --ignore-installed --prefix=%s %s" % (self.installdir, whl_paths[0])
# if extensions are listed, assume they will provide all required dependencies,
# so use --no-deps to prevent pip from downloading & installing them
if self.cfg['exts_list']:
cmd += ' --no-deps'
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s", whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341 on Python < 3.3
# If the site-packages/google/__init__.py file is missing, make it an empty file.
# This fixes the "No module named google.protobuf" error that sometimes shows up during sanity_check
# For Python >= 3.3 the logic is reversed: The __init__.py must not exist.
# See e.g. http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html
google_protobuf_dir = os.path.join(self.installdir, self.pylibdir, 'google', 'protobuf')
google_init_file = os.path.join(self.installdir, self.pylibdir, 'google', '__init__.py')
if LooseVersion(det_python_version(self.python_cmd)) < LooseVersion('3.3'):
if os.path.isdir(google_protobuf_dir) and not is_readable(google_init_file):
self.log.debug("Creating (empty) missing %s", google_init_file)
write_file(google_init_file, '')
else:
if os.path.exists(google_init_file):
self.log.debug("Removing %s for Python >= 3.3", google_init_file)
remove_file(google_init_file)
# Fix cuda header paths
# This is needed for building custom TensorFlow ops
if LooseVersion(self.version) < LooseVersion('1.14'):
pyshortver = '.'.join(get_software_version('Python').split('.')[:2])
regex_subs = [(r'#include "cuda/include/', r'#include "')]
base_path = os.path.join(self.installdir, 'lib', 'python%s' % pyshortver, 'site-packages', 'tensorflow',
'include', 'tensorflow')
for header in glob.glob(os.path.join(base_path, 'stream_executor', 'cuda', 'cuda*.h')) + glob.glob(
os.path.join(base_path, 'core', 'util', 'cuda*.h')):
apply_regex_substitutions(header, regex_subs)
def get_cpu_model():
"""
Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
"""
model = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_CPUINFO_FP):
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
arch = get_cpu_architecture()
if arch in [AARCH32, AARCH64]:
# On ARM platforms, no model name is provided in /proc/cpuinfo. However, for vanilla ARM cores
# we can reverse-map the part number.
vendor = get_cpu_vendor()
if vendor == ARM:
model_regex = re.compile(r"CPU part\s+:\s*(\S+)", re.M)
# There can be big.LITTLE setups with different types of cores!
model_ids = model_regex.findall(proc_cpuinfo)
if model_ids:
id_list = []
for model_id in sorted(set(model_ids)):
id_list.append(ARM_CORTEX_IDS.get(model_id, UNKNOWN))
model = vendor + ' ' + ' + '.join(id_list)
_log.debug(
"Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern, PROC_CPUINFO_FP, model)
else:
# we need 'model name' on Linux/x86, but 'model' is there first with different info
# 'model name' is not there for Linux/POWER, but 'model' has the right info
model_regex = re.compile(
r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M)
res = model_regex.search(proc_cpuinfo)
if res is not None:
model = res.group('model').strip()
_log.debug(
"Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern, PROC_CPUINFO_FP, model)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
out, ec = run_cmd(cmd,
force_in_dry_run=True,
trace=False,
stream_output=False)
if ec == 0:
model = out.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" %
(cmd, model))
if model is None:
model = UNKNOWN
_log.warning("Failed to determine CPU model, returning %s" % model)
return model
def get_cpu_speed():
"""
Returns the (maximum) cpu speed in MHz, as a float value.
In case of throttling, the highest cpu speed is returns.
"""
cpu_freq = None
os_type = get_os_type()
if os_type == LINUX:
# Linux with cpu scaling
if is_readable(MAX_FREQ_FP):
_log.debug("Trying to determine CPU frequency on Linux via %s" % MAX_FREQ_FP)
txt = read_file(MAX_FREQ_FP)
cpu_freq = float(txt) / 1000
# Linux without cpu scaling
elif is_readable(PROC_CPUINFO_FP):
_log.debug("Trying to determine CPU frequency on Linux via %s" % PROC_CPUINFO_FP)
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
# 'cpu MHz' on Linux/x86 (& more), 'clock' on Linux/POWER
cpu_freq_regex = re.compile(r"^(?:cpu MHz|clock)\s*:\s*(?P<cpu_freq>\d+(?:\.\d+)?)", re.M)
res = cpu_freq_regex.search(proc_cpuinfo)
if res:
cpu_freq = float(res.group('cpu_freq'))
_log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex.pattern, cpu_freq))
else:
_log.debug("Failed to determine CPU frequency from %s", PROC_CPUINFO_FP)
else:
_log.debug("%s not found to determine max. CPU clock frequency without CPU scaling", PROC_CPUINFO_FP)
elif os_type == DARWIN:
cmd = "sysctl -n hw.cpufrequency_max"
_log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
# returns clock frequency in cycles/sec, but we want MHz
cpu_freq = float(out.strip()) / (1000 ** 2)
else:
raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type)
return cpu_freq
def get_cpu_vendor():
"""
Try to detect the CPU vendor
:return: a value from the CPU_VENDORS list
"""
vendor = None
os_type = get_os_type()
if os_type == LINUX:
vendor_regex = None
arch = get_cpu_architecture()
if arch == X86_64:
vendor_regex = re.compile(r"vendor_id\s+:\s*(\S+)")
elif arch == POWER:
vendor_regex = re.compile(r"model\s+:\s*((\w|-)+)")
elif arch in [AARCH32, AARCH64]:
vendor_regex = re.compile(r"CPU implementer\s+:\s*(\S+)")
if vendor_regex and is_readable(PROC_CPUINFO_FP):
vendor_id = None
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
res = vendor_regex.search(proc_cpuinfo)
if res:
vendor_id = res.group(1)
if vendor_id in VENDOR_IDS:
vendor = VENDOR_IDS[vendor_id]
_log.debug(
"Determined CPU vendor on Linux as being '%s' via regex '%s' in %s",
vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
out, ec = run_cmd(cmd,
force_in_dry_run=True,
trace=False,
stream_output=False)
out = out.strip()
if ec == 0 and out in VENDOR_IDS:
vendor = VENDOR_IDS[out]
_log.debug(
"Determined CPU vendor on DARWIN as being '%s' via cmd '%s" %
(vendor, cmd))
if vendor is None:
vendor = UNKNOWN
_log.warning("Could not determine CPU vendor on %s, returning %s" %
(os_type, vendor))
return vendor
def get_cpu_vendor():
"""
Try to detect the CPU vendor
:return: a value from the CPU_VENDORS list
"""
vendor = None
os_type = get_os_type()
if os_type == LINUX:
vendor_regex = None
arch = get_cpu_architecture()
if arch == X86_64:
vendor_regex = re.compile(r"vendor_id\s+:\s*(\S+)")
elif arch == POWER:
vendor_regex = re.compile(r"model\s+:\s*(\w+)")
elif arch in [AARCH32, AARCH64]:
vendor_regex = re.compile(r"CPU implementer\s+:\s*(\S+)")
if vendor_regex and is_readable(PROC_CPUINFO_FP):
vendor_id = None
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
res = vendor_regex.search(proc_cpuinfo)
if res:
vendor_id = res.group(1)
if vendor_id in VENDOR_IDS:
vendor = VENDOR_IDS[vendor_id]
_log.debug(
"Determined CPU vendor on Linux as being '%s' via regex '%s' in %s",
vendor,
vendor_regex.pattern,
PROC_CPUINFO_FP,
)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
out, ec = run_cmd(cmd, force_in_dry_run=True)
out = out.strip()
if ec == 0 and out in VENDOR_IDS:
vendor = VENDOR_IDS[out]
_log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
if vendor is None:
vendor = UNKNOWN
_log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor))
return vendor
def install_step(self):
"""Custom install procedure for TensorFlow."""
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(
os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if len(whl_paths) == 1:
# --upgrade is required to ensure *this* wheel is installed
# cfr. https://github.com/tensorflow/tensorflow/issues/7449
cmd = "pip install --ignore-installed --prefix=%s %s" % (
self.installdir, whl_paths[0])
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s",
whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341
# If the site-packages/google/__init__.py file is missing, make
# it an empty file.
# This fixes the "No module named google.protobuf" error that
# sometimes shows up during sanity_check
google_init_file = os.path.join(self.installdir, self.pylibdir,
'google', '__init__.py')
if not is_readable(google_init_file):
write_file(google_init_file, '')
# test installation using MNIST tutorial examples
# (can't be done in sanity check because mnist_deep.py is not part of installation)
if self.cfg['runtest']:
pythonpath = os.getenv('PYTHONPATH', '')
env.setvar(
'PYTHONPATH', '%s:%s' %
(os.path.join(self.installdir, self.pylibdir), pythonpath))
for mnist_py in ['mnist_softmax.py', 'mnist_with_summaries.py']:
tmpdir = tempfile.mkdtemp(suffix='-tf-%s-test' %
os.path.splitext(mnist_py)[0])
mnist_py = os.path.join(self.start_dir, 'tensorflow',
'examples', 'tutorials', 'mnist',
mnist_py)
cmd = "%s %s --data_dir %s" % (self.python_cmd, mnist_py,
tmpdir)
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def get_cpu_features():
"""
Get list of CPU features
"""
cpu_feat = []
os_type = get_os_type()
if os_type == LINUX:
if is_readable(PROC_CPUINFO_FP):
_log.debug("Trying to determine CPU features on Linux via %s", PROC_CPUINFO_FP)
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
# 'flags' on Linux/x86, 'Features' on Linux/ARM
flags_regex = re.compile(r"^(?:flags|[fF]eatures)\s*:\s*(?P<flags>.*)", re.M)
res = flags_regex.search(proc_cpuinfo)
if res:
cpu_feat = sorted(res.group('flags').lower().split())
_log.debug("Found CPU features using regex '%s': %s", flags_regex.pattern, cpu_feat)
elif get_cpu_architecture() == POWER:
# for Linux@POWER systems, no flags/features are listed, but we can check for Altivec
cpu_altivec_regex = re.compile("^cpu\s*:.*altivec supported", re.M)
if cpu_altivec_regex.search(proc_cpuinfo):
cpu_feat.append('altivec')
# VSX is supported since POWER7
cpu_power7_regex = re.compile("^cpu\s*:.*POWER(7|8|9)", re.M)
if cpu_power7_regex.search(proc_cpuinfo):
cpu_feat.append('vsx')
else:
_log.debug("Failed to determine CPU features from %s", PROC_CPUINFO_FP)
else:
_log.debug("%s not found to determine CPU features", PROC_CPUINFO_FP)
elif os_type == DARWIN:
for feature_set in ['extfeatures', 'features', 'leaf7_features']:
cmd = "sysctl -n machdep.cpu.%s" % feature_set
_log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd)
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
cpu_feat.extend(out.strip().lower().split())
cpu_feat.sort()
else:
raise SystemToolsException("Could not determine CPU features (OS: %s)" % os_type)
return cpu_feat
