def install(self, cores=None):
"""Installs OpenMPI.
Args:
cores (int, optional): The number of cores on the system.
Returns:
Boolean: True if installation was successful otherwise False.
"""
if cores is None:
cores = 1
build_dir = self.mpi_dir + "/build"
bin_loc = build_dir + "/bin/mpicc"
if os.path.isfile(bin_loc):
return True
if not os.path.isdir(build_dir):
prettify.error_message(
'Cannot install OpenMPI because "{}" could not be found.'.
format(self.mpi_dir))
return False
logging.info("Installing OpenMPI using %d Make threads.", cores)
execute.output("sudo -E make -s -j {} install".format(cores),
build_dir)
if os.path.exists(bin_loc):
return True
return False
def yum(packages):
"""Install yum packages.
Args:
packages (list): Yum packages to install.
"""
try:
if not shutil.which("yum"):
return
logging.info('Installing prerequisites using "yum".')
devtools = "sudo -E yum groupinstall -y --skip-broken " '"Development Tools"'
logging.debug("Yum install: %s", devtools)
output = execute.output(devtools)
logging.debug("Yum output: %s", output)
for package in packages:
cmd = "sudo -E yum install -y --skip-broken " + package
logging.debug("Yum install: %s", cmd)
output = execute.output(cmd)
logging.debug("Yum output: %s", output)
except IOError as err:
logging.debug(err)
except ValueError as err:
logging.debug(err)
except TypeError as err:
logging.debug(err)
def numa_nodes():
"""The number of NUMA nodes.
Example:
>>> numa_nodes()
1
Returns:
Integer: NUMA nodes.
"""
nodes = None
try:
if shutil.which("numactl"):
stdout = execute.output("numactl --hardware | grep -c cpus")
if stdout:
nodes = int(stdout.rstrip())
if not nodes and shutil.which("dmesg"):
stdout = execute.output(
r'dmesg | grep -c "NUMA node\|No NUMA configuration found"')
if stdout:
nodes = int(stdout.rstrip())
return nodes
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def setup(self):
"""Extract MySQL.
Returns:
Boolean: True if extraction was successful otherwise False.
"""
files_dir = self.mysql_dir + "/mysql-files"
if not os.path.isdir(self.mysql_dir):
prettify.error_message(
'Cannot setup MySQL because "{}" could not be found.'.format(
self.mysql_dir))
return False
if os.path.isdir(files_dir) and os.listdir(files_dir):
return True
os.makedirs(files_dir, exist_ok=True)
os.chmod(files_dir, 0o750)
execute.output(
"./bin/mysqld --initialize-insecure --user=root --basedir={} "
"--datadir={}".format(self.mysql_dir, files_dir),
working_dir=self.mysql_dir,
)
execute.output(
"./bin/mysql_ssl_rsa_setup --user=root --basedir={} --datadir={}".
format(self.mysql_dir, files_dir),
working_dir=self.mysql_dir,
)
return True
def install(self, cores=None):
"""Installs glibc.
Args:
cores (int, optional): The number of cores on the system.
Returns:
Boolean: True if installation was successful otherwise False.
"""
if cores is None:
cores = 1
bin_loc = "/usr/local/glibc/lib/ld-{}.so".format(self.version)
build_dir = self.glibc_dir + "/build"
if os.path.isfile(bin_loc):
return True
if not os.path.isdir(build_dir):
prettify.error_message(
'Cannot install glibc because "{}" could not be found.'.format(
build_dir))
return False
logging.info("Installing glibc using %d Make threads.", cores)
execute.output("sudo -E make -j {} install".format(cores), build_dir)
if os.path.isfile(bin_loc):
return True
return False
def build(self, threads, cores=None, cflags=None, avx512=None):
"""Compiles OpenBLAS.
Args:
threads (int): The number of threads on the system.
cores (int, optional): The number of cores on the system.
cflags (str, optional): The CFLAGS for GCC.
avx512 (bool, optional): Whether to enable AVX-512 CFLAGS.
Returns:
Boolean: True if compilation was successful otherwise False.
"""
if cores is None:
cores = 1
if cflags is None:
cflags = "-march=native -mtune=native"
if "-O" not in cflags:
cflags += " -O3 "
if avx512 is True:
cflags += (" -mavx512f -mavx512cd -mavx512bw -mavx512dq -mavx512vl"
" -mavx512ifma -mavx512vbmi ")
bin_loc = self.openblas_dir + "/libopenblas.so"
shell_env = os.environ.copy()
shell_env["CFLAGS"] = cflags
shell_env["OMP_NUM_THREADS"] = str(threads)
openmpi_dir = self.src_dir + "/openmpi/build/bin"
mpifort_bin = openmpi_dir + "/mpifort"
mpicc_bin = openmpi_dir + "/mpicc"
if os.path.isfile(bin_loc):
return True
if not os.path.isdir(self.openblas_dir):
prettify.error_message(
'Cannot compile OpenBLAS because "{}" could not be found.'.
format(self.openblas_dir))
return False
logging.info(
"Compiling OpenBLAS using %d OMP threads, %d Make threads, "
'and "%s" CFLAGS.',
threads,
cores,
cflags,
)
execute.output(
"make -j {} FC={} CC={} USE_OPENMP=1 USE_THREAD=1".format(
cores, mpifort_bin, mpicc_bin),
self.openblas_dir,
environment=shell_env,
)
if os.path.isfile(bin_loc):
return True
return False
def build(self, cores=None, cflags=None):
"""Compiles glibc.
Args:
cores (int, optional): The number of cores on the system.
cflags (str, optional): The CFLAGS for GCC.
Returns:
Boolean: True if compilation was successful otherwise False.
"""
if cores is None:
cores = 1
if cflags is None:
cflags = ""
# `-ffast-math` cannot be used to compile glibc
if "-Ofast" in cflags:
cflags = cflags.replace("-Ofast", "")
if "-ffast-math" in cflags:
cflags = cflags.replace("-ffast-math", "")
# `-O3` fails sometimes
if "-O3" in cflags:
cflags = cflags.replace("-O3", "")
# Optimizations are needed for glibc
if "-O" not in cflags:
cflags += " -O2 "
shell_env = os.environ.copy()
shell_env["CFLAGS"] = cflags
build_dir = self.glibc_dir + "/build"
bin_loc = build_dir + "/libc.so"
if os.path.isfile(bin_loc):
return True
if not os.path.isdir(self.glibc_dir):
prettify.error_message(
'Cannot compile glibc because "{}" could not be found.'.format(
self.glibc_dir))
return False
logging.info('Compiling glibc using %d Make threads and "%s" CFLAGS',
cores, cflags)
os.makedirs(build_dir, exist_ok=True)
execute.output(
"spet.lib./configure --prefix=/usr/local/glibc",
build_dir,
environment=shell_env,
)
execute.output("make -j " + str(cores),
build_dir,
environment=shell_env)
if os.path.isfile(bin_loc):
return True
return False
def disable_swap():
"""Disable swap."""
try:
if shutil.which("swapoff"):
execute.output("sudo swapoff -a")
return True
return False
except IOError as err:
logging.debug(err)
def __image_built(self, name, env=None):
"""Check if the named image is built.
Notes:
* Requires Docker daemon (`dockerd`) to be running.
Args:
name (str): The name of the image.
env (dict): The shell environment exports.
Returns:
Boolean: True if image found otherwise False.
"""
if env is None:
env = os.environ.copy()
env["PATH"] = self.docker_dir + ":" + env["PATH"]
logging.debug("Checking if Docker image is built.")
image_output = execute.output("docker images",
working_dir=self.docker_dir,
environment=env)
found_images = grep.text(image_output, name)
if found_images:
return True
return False
def version():
"""The default Java version.
Example:
>>> version()
'1.8.0_151'
Returns:
String: The detected version or the string "Unknown".
"""
try:
java_ver = execute.output("java -version")
java_ver = java_ver.rstrip().split("\n")[0]
if java_ver:
pattern = r"(\w+?)\sversion\s\"(.+?)\""
sub = r"\1-\2"
java_ver = re.sub(pattern, sub, java_ver)
return java_ver
return "Unknown"
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def version():
"""The default GCC version.
Example:
>>> version()
'6.2.0'
Returns:
String: The detected version or the string "Unknown".
"""
try:
gcc_ver = execute.output("gcc --version")
gcc_ver = gcc_ver.rstrip().split("\n")[0]
if gcc_ver:
pattern = r"gcc\s\(.*\)\s([.0-9]+).*"
sub = r"\1"
gcc_ver = re.sub(pattern, sub, gcc_ver)
return gcc_ver
return "Unknown"
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def prerun():
"""Clear up system resources before running a benchmark."""
try:
cleared = False
if shutil.which("sync"):
execute.output("sudo sync")
cleared = True
drop_caches = "/proc/sys/vm/drop_caches"
if os.path.isfile(drop_caches):
file.write(drop_caches, "3")
cleared = True
return cleared
except ValueError as err:
logging.debug(err)
except IOError as err:
logging.debug(err)
def flags():
"""The detected `-march=|-mcpu=` and `-mtune=` flags for GCC.
Example:
>>> flags()
'-march=native -march=native'
Returns:
A string of march and mtune.
march (str): The complete flag for `-march=` or `-mcpu=`.
mtune (str): The complete flag for `-mtune=`.
"""
try:
march_mcpu = "march"
cpu_name = processor.name().lower()
if "power" in cpu_name or "ppc" in cpu_name:
march_mcpu = "mcpu"
march_output = execute.output(
"gcc -{}=native -Q --help=target".format(march_mcpu))
march_flag = grep.text(march_output, "-{}=".format(march_mcpu))
march_flag = march_flag[0].rstrip().split()[1].strip()
if "native" in march_flag or not march_flag:
march_flag = "native"
mtune_output = execute.output(
"gcc -{}={} -mtune=native -Q --help=target".format(
march_mcpu, march_flag))
mtune_flag = grep.text(mtune_output, "-mtune=")
mtune_flag = mtune_flag[0].rstrip().split()[1].strip()
if "native" in mtune_flag or not mtune_flag:
mtune_flag = "native"
march = "-{}={}".format(march_mcpu, march_flag)
mtune = "-mtune=" + mtune_flag
return march + " " + mtune
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def build(self, cores=None, cflags=None):
"""Compiles zlib.
Args:
cores (int, optional): The number of cores on the system.
cflags (str, optional): The CFLAGS for GCC.
Returns:
Boolean: True if compilation was successful otherwise False.
"""
if cores is None:
cores = 1
if cflags is None:
cflags = "-march=native -mtune=native"
if "-O" not in cflags:
cflags += " -O3 "
bin32_loc = self.zlib_dir + "/minigzip"
bin64_loc = self.zlib_dir + "/minigzip64"
shell_env = os.environ.copy()
shell_env["CFLAGS"] = cflags
logging.debug("CFLAGS: %s", shell_env["CFLAGS"])
if os.path.isfile(bin32_loc) or os.path.isfile(bin64_loc):
return True
if not os.path.isdir(self.zlib_dir):
prettify.error_message('Cannot compile zlib because "{}" could '
"not be found.".format(self.zlib_dir))
return False
logging.info('Compiling zlib with %d Make threads, and "%s" CFLAGS.',
cores, cflags)
cmd = "./configure && make -j " + str(cores)
self.commands.append("Build: CFLAGS = " + cflags)
self.commands.append("Build: " + cmd)
execute.output(cmd, self.zlib_dir, environment=shell_env)
if not os.path.isfile(bin32_loc) or not os.path.isfile(bin64_loc):
return False
return True
def build(self, cores=None, cflags=None):
"""Compiles OpenMPI.
Args:
cores (int, optional): The number of cores on the system.
cflags (str, optional): The CFLAGS for GCC.
Returns:
Boolean: True if compilation was successful otherwise False.
"""
if cores is None:
cores = 1
if cflags is None:
cflags = "-march=native -mtune=native"
if "-O" not in cflags:
cflags += " -O3 "
build_dir = self.mpi_dir + "/build"
bin_loc = build_dir + "/bin/mpicc"
shell_env = os.environ.copy()
shell_env["CFLAGS"] = cflags
if os.path.isfile(bin_loc):
return True
if not os.path.isdir(self.mpi_dir):
prettify.error_message(
'Cannot compile OpenMPI because "{}" could not be found.'.
format(self.mpi_dir))
return False
logging.info(
'Compiling OpenMPI using %d Make threads and "%s" CFLAGS.', cores,
cflags)
os.makedirs(build_dir, exist_ok=True)
execute.output("../configure --prefix=" + build_dir,
build_dir,
environment=shell_env)
execute.output("make -s -j {} all".format(cores),
build_dir,
environment=shell_env)
return True
def frequency():
"""The detected memory frequency.
Example:
>>> frequency()
2666
Returns:
Integer: The memory frequency in MHz.
"""
dimm_freq = None
freq = None
try:
if shutil.which("dmidecode"):
output = None
dmidecode_output = execute.output("dmidecode -t memory")
outputs = grep.text(dmidecode_output, r"^\s*Speed:")
for dimm in outputs:
if "Unknown" in dimm:
continue
output = dimm
break
if output:
dimm_freq = output.strip().split()[1]
if not dimm_freq and shutil.which("lshw"):
lshw_output = execute.output("lshw -short -C memory")
dimms = grep.text(lshw_output, "DIMM")
if dimms:
dimm_freq = dimms[0].strip().split()[6]
if "." in dimm_freq:
freq = int(float(dimm_freq))
elif dimm_freq and dimm_freq.isdigit():
freq = int(dimm_freq)
return freq
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def nofiles():
"""Sets the number of files limit."""
limits_conf = "/etc/security/limits.conf"
sysctl_conf = "/etc/sysctl.conf"
try:
if os.path.isfile(limits_conf):
lines = grep.file(limits_conf, "nofile 1048576")
if not lines:
file.write(limits_conf, "* - nofile 1048576", append=True)
if os.path.isfile(sysctl_conf):
lines = grep.file(sysctl_conf, "fs.file-max = 1048576")
if not lines:
file.write(sysctl_conf, "fs.file-max = 1048576", append=True)
if shutil.which("sysctl"):
execute.output("sudo sysctl -p")
return True
except ValueError as err:
logging.debug(err)
except IOError as err:
logging.debug(err)
def frequency():
"""The detected processor frequency.
Example:
>>> frequency()
3200
Returns:
Integer: The processor frequency in MHz.
"""
try:
mhz_freq = None
freq = None
if shutil.which("dmidecode"):
dmidecode_output = execute.output("dmidecode -t processor")
dmidecode_output = grep.text(dmidecode_output, "Max Speed")
if dmidecode_output:
mhz_freq = dmidecode_output[0].strip().split()[2]
elif shutil.which("lscpu"):
lscpu_output = execute.output("lscpu")
lscpu_output = grep.text(lscpu_output, "CPU max MHz:")
if lscpu_output:
mhz_freq = lscpu_output[0].strip().split()[3]
elif os.path.isfile("/proc/cpuinfo"):
cpuinfo_output = grep.file("/proc/cpuinfo", "cpu MHz")
if cpuinfo_output:
mhz_freq = cpuinfo_output[0].strip().split()[3]
if "." in mhz_freq:
freq = int(float(mhz_freq))
elif mhz_freq and mhz_freq.isdigit():
freq = int(mhz_freq)
return freq
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def distribution():
"""The detected operating system distribution information.
Example:
>>> distribution()
('Ubuntu', '17.04')
Returns:
A tuple of (distribution_name, distribution_version).
distribution_name (str): The operating system name.
distribution_version (str): The operating system version.
"""
try:
if os.path.isfile("/etc/os-release"):
distribution_name = execute.output(
r'. /etc/os-release && echo "${NAME}"')
distribution_version = execute.output(
r'. /etc/os-release && echo "${VERSION}"')
elif os.path.isfile("/etc/lsb-release"):
distribution_name = execute.output(
r'. /etc/lsb-release && echo "${DISTRIB_ID}"')
distribution_version = execute.output(
r'. /etc/lsb-release && echo "${DISTRIB_RELEASE}"')
elif os.path.isfile("/etc/debian_version"):
distribution_name = "Debian"
distribution_version = file.read("/etc/debian_version")
elif os.path.isfile("/etc/redhat-release"):
distribution_name = "Redhat"
distribution_version = file.read("/etc/redhat-release")
else:
distribution_name = os.uname().sysname # pylint: disable=E1101
distribution_version = os.uname().release # pylint: disable=E1101
return distribution_name.strip(), distribution_version.strip()
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def total():
"""The total system memory in GB.
Examples:
>>> total()
64
Returns:
Integer: RAM in GB.
"""
try:
ram_gb = None
ram_kb = None
if shutil.which("lshw"):
lshw_output = (execute.output(
"lshw -class memory | grep -A 9 'bank:' | awk '/size:/ "
"{print $2}'").rstrip().split("\n"))
# lshw is in binary prefixed notation; however, it is actually
# usually SI prefixed. Will do binary prefixed (kibibyte KiB)
# conversion to SI prefixed (kilobyte KB); then at the end, do
# rounding the the nearest n^2 to handle inconsistencies.
for dimm in lshw_output:
if "GiB" in dimm:
dimm = re.sub(r"[^0-9]", "", dimm)
ram_gb += int(dimm)
elif "Mib" in dimm:
dimm = re.sub(r"[^0-9]", "", dimm)
dimm = int(dimm) * 1024
ram_gb += __closest_power_of_two(dimm)
elif "Kib" in dimm:
dimm = re.sub(r"[^0-9]", "", dimm)
dimm = int(dimm) * 1024 * 1024
ram_gb += __closest_power_of_two(dimm)
if not ram_gb and os.path.isfile("/proc/meminfo"):
meminfo_output = grep.file("/proc/meminfo", "MemTotal")
ram_kb = re.sub("MemTotal:", "", meminfo_output[0])
ram_kb = re.sub("kB", "", ram_kb)
ram_kb = ram_kb.strip().split()[0]
# 1024/1000 seems to work instead of either
# a) 1000/1000
# b) 1024/1024
ram_gb = int(ram_kb) / 1024 / 1000
return ram_gb
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def setup(self, cores=None, cflags=None):
"""Setup the Linux kernel config file.
Args:
cores (int, optional): The number of cores on the system.
cflags (str, optional): The CFLAGS for GCC.
Returns:
Boolean: True if setup was successful otherwise False.
"""
if cores is None:
cores = 1
if cflags is None:
cflags = "-march=native -mtune=native"
config_loc = self.kernel_dir + "/.config"
shell_env = os.environ.copy()
if "-O" not in cflags:
cflags += " -O3 "
shell_env["CFLAGS"] = cflags
if os.path.isfile(config_loc):
return True
if not os.path.isdir(self.kernel_dir):
prettify.error_message(
'Cannot configure the Linux kernel because "{}" could not be'
" found.".format(self.kernel_dir))
return False
logging.info(
"Setting up the Linux kernel with %d Make threads, "
'and "%s" CFLAGS.',
cores,
str(shell_env["CFLAGS"]),
)
cmd = "make -s -j {0} defconfig && make -s -j {0} clean".format(cores)
output = execute.output(cmd,
working_dir=self.kernel_dir,
environment=shell_env)
logging.debug("Build output:\n%s", output)
self.commands.append("Setup: CFLAGS = " + cflags)
self.commands.append("Setup: " + cmd)
if os.path.isfile(config_loc):
return True
return False
def architecture():
"""The system architecture.
Example:
>>> architecture()
(64, 'x86_64')
Returns:
A tuple of (bits, type).
bits (int): The architecture bit version (32 or 64).
type (str): The machine type.
"""
try:
first_pattern = r"architecture:\s*"
second_pattern = r"/x86_"
third_pattern = r"i[3-6]86"
machine_type = os.uname().machine # pylint: disable=E1101
if shutil.which("lscpu"):
lscpu_output = execute.output("lscpu")
arch = grep.text(lscpu_output, "Architecture:")
arch = re.sub(first_pattern, "", arch[0])
arch = re.sub(second_pattern, "", arch)
arch = re.sub(third_pattern, "32", arch)
if not arch:
arch = machine_type
arch = re.sub(second_pattern, "", arch)
arch = re.sub(third_pattern, "32", arch)
arch = arch.lower()
if "arm" in arch:
arm_ver = re.sub("armv", "", arch)
if int(arm_ver) >= 8:
arch = 64
else:
arch = 32
if "64" in arch:
arch = 64
return int(arch), machine_type
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def install(self):
"""Install MKL.
Returns:
Boolean: True if installation was successful otherwise False.
"""
if os.path.isdir("/opt/intel/mkl"):
return True
if not os.path.isdir(self.mkl_dir):
prettify.error_message(
'Cannot install MKL because "{}" could not be found.'.format(
self.mkl_dir))
return False
logging.info("Installing MKL.")
execute.output('{}/install.sh --silent "{}/provided/silent.cfg"'.format(
self.mkl_dir, self.src_dir))
if os.path.isdir(self.mkl_dir):
return True
return False
def name():
"""The detected processor name.
Example:
>>> name()
'Intel Core(TM) i7-7700 CPU'
Returns:
String: Processor name.
"""
cpu_name = None
try:
if os.path.exists("/proc/cpuinfo"):
cpu_name = grep.file("/proc/cpuinfo", r"^model name\s*:\s*")
logging.debug("/proc/cpuinfo model name: %s", str(cpu_name))
if cpu_name:
cpu_name = " ".join(cpu_name[0].strip().split()[3:])
return cpu_name
if not shutil.which("lscpu"):
return None
lscpu_output = execute.output("lscpu")
cpu_name = grep.text(lscpu_output, "Model name:")
logging.debug("lscpu model name: %s", str(cpu_name))
if not cpu_name:
return None
cpu_name = " ".join(cpu_name[0].strip().split()[2:])
if cpu_name:
return cpu_name
cpu_name = grep.text(lscpu_output, "CPU:")
cpu_name = " ".join(cpu_name[0].strip().split()[1:])
return cpu_name
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def run(self):
"""Run MLC three times.
Returns:
If success: A dict containing (unit, run1, run2, run3, average,
median).
unit (str): Latency units.
run1 (list): Latency for each NUMA node of the first run.
run2 (list): Latency for each NUMA node of the second run.
run3 (list): Latency for each NUMA node of the third run.
average (list): Average for each NUMA node of run1, run2, and
run3.
median (list): Median for each NUMA node of run1, run2, and
run3.
If error: A dict containing (error).
error (str): Error message.
"""
bin_loc = self.mlc_dir + "/Linux/mlc_avx512"
cmd = "modprobe msr; {} --latency_matrix".format(bin_loc)
results = {"unit": "ns"}
if not os.path.isfile(bin_loc):
text = 'Cannot run MLC because "{}" could not be found.'.format(
bin_loc)
prettify.error_message(text)
return {"error": text}
os.makedirs(self.results_dir, exist_ok=True)
self.commands.append("Run: " + cmd)
output = execute.output(cmd, self.mlc_dir)
file.write(self.results_dir + "/mlc_output.txt", output)
found_lines = grep.text(output, r"^\s*0")
if found_lines:
node_latencies = found_lines[0].strip().split()
node_latencies.pop(0) # Remove leading '0' for first node
for index, latency in enumerate(node_latencies):
node_latencies[index] = float(latency)
results["latencies"] = node_latencies
logging.info("MLC results: %s", str(results))
return results
def node_topology():
"""The arrangement of physical cores to logical cores.
Example:
>>> node_topology()
([0, 2], [1, 4])
Returns:
A tuple of (cores, threads, sockets).
physical_cores (list): The physical core CPU IDs.
logical_cores (list): The logical core CPU IDs.
"""
physical_cores = None
logical_cores = None
try:
stdout = execute.output("grep 'physical id' /proc/cpuinfo").strip()
lines = stdout.split("\n")
if not lines:
return physical_cores, logical_cores
physical_cores = []
logical_cores = []
for core_id, line in enumerate(lines, start=0):
# e.g., "physical id\t: 0"
core = line.split()[3]
if int(core) % 2 == 0:
physical_cores.append(str(core_id))
continue
logical_cores.append(str(core_id))
return physical_cores, logical_cores
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
def zypper(packages):
"""Install zypper packages.
Args:
packages (list): Zypper packages to install.
"""
try:
if not shutil.which("zypper"):
return False
logging.info('Installing prerequisites using "zypper".')
for package in packages:
cmd = "sudo -E zypper install -l -y --force-resolution " + package
logging.debug("Zypper install: %s", cmd)
output = execute.output(cmd)
logging.debug("Zypper output: %s", output)
except IOError as err:
logging.debug(err)
except ValueError as err:
logging.debug(err)
except TypeError as err:
logging.debug(err)
def apt_get(packages):
"""Install apt-get packages.
Args:
packages (list): Apt-get packages to install.
"""
try:
if not shutil.which("apt-get"):
return
logging.info('Installing prerequisites using "apt-get".')
for package in packages:
cmd = "sudo -E apt-get install -y --ignore-missing " + package
logging.debug("Apt-get install: %s", cmd)
output = execute.output(cmd)
logging.debug("Apt-get output: %s", output)
except IOError as err:
logging.debug(err)
except ValueError as err:
logging.debug(err)
except TypeError as err:
logging.debug(err)
def aptitude(packages):
"""Install aptitude packages.
Args:
packages (list): Aptitude packages to install.
"""
try:
if not shutil.which("aptitude"):
logging.error("The aptitude package manager could not be found.")
return
logging.info('Installing prerequisites using "aptitude".')
for package in packages:
cmd = "sudo -E aptitude install -y --ignore-missing " + package
logging.debug("Aptitude install: %s", cmd)
output = execute.output(cmd)
logging.debug("Aptitude output: %s", output)
except IOError as err:
logging.debug(err)
except ValueError as err:
logging.debug(err)
except TypeError as err:
logging.debug(err)
def topology():
"""The processor topology.
Examples:
>>> topology()
(4, 8, 1)
Returns:
A tuple of (cores, threads, sockets).
cores (int): The number of physical processors.
threads (int): The number of logical processors.
sockets (int): The number of processor sockets.
"""
cores = None
sockets = None
threads_per_core = None
threads = None
cores_per_processor = None
try:
if shutil.which("lscpu"):
lscpu_output = execute.output("lscpu")
sockets = grep.text(lscpu_output, "Socket")
sockets = re.sub(r"Socket\(s\):\s*", "", sockets[0])
sockets = int(sockets.strip())
threads_per_core = grep.text(lscpu_output,
r"Thread\(s\) per core:")
threads_per_core = re.sub(r"Thread\(s\) per core:\s*", "",
threads_per_core[0])
threads_per_core = int(threads_per_core.strip())
cores_per_processor = grep.text(lscpu_output,
r"Core\(s\) per socket:")
cores_per_processor = re.sub(r"Core\(s\) per socket:\s*", "",
cores_per_processor[0])
cores_per_processor = int(cores_per_processor.strip())
if not sockets and shutil.which("dmidecode"):
dmidecode_output = execute.output("dmidecode -t 4")
sockets = len(grep.text(dmidecode_output, "Socket Designation"))
total_threads = grep.text(dmidecode_output, r"Thread Count\:")
total_threads = re.sub(r"Thread Count:", "", total_threads[0])
total_threads = total_threads.strip().split()[0]
total_threads = int(total_threads)
cores_per_processor = grep.text(dmidecode_output, r"Core Count\:")
cores_per_processor = re.sub(r"Core Count:\s*", "",
cores_per_processor[0])
cores_per_processor = cores_per_processor.strip().split()[0]
cores_per_processor = int(cores_per_processor)
threads_per_core = total_threads / cores_per_processor
if not sockets:
thread_siblings = (file.read(
"/sys/devices/system/cpu/cpu1/topology/thread_siblings_list").
strip().split(","))
threads_per_core = 1
if shutil.which("nproc"):
total_threads = execute.output("nproc --all")
elif os.path.isfile("/proc/cpuinfo"):
total_threads = len(grep.file("/proc/cpuinfo", r"^processor"))
else:
total_threads = execute.output("getconf _NPROCESSORS_ONLN")
total_threads = int(total_threads)
if len(thread_siblings) > 1:
threads_per_core = 2
if total_threads:
cores_per_processor = total_threads / threads_per_core
if not sockets:
raise Exception("The number of sockets was not found.")
if not cores_per_processor:
raise Exception("The number of cores per processor was not found.")
if not threads_per_core:
raise Exception("The number of threads per core was not found.")
cores = sockets * cores_per_processor
threads = threads_per_core * cores
return cores, threads, sockets
except IOError as err:
logging.error(err)
except ValueError as err:
logging.error(err)
except TypeError as err:
logging.error(err)
