def get_syspaths(cls):
if cls.syspaths is not None:
return cls.syspaths
if config.standard_system_paths:
cls.syspaths = config.standard_system_paths
return cls.syspaths
# detect system paths using registry
def gen_expected_regex(parts):
whitespace = r"[\s]+"
return whitespace.join(parts)
# TODO: Research if there is an easier way to pull system PATH from
# registry in powershell
paths = []
cmd = [
"REG", "QUERY",
("HKLM\\SYSTEM\\CurrentControlSet\\"
"Control\\Session Manager\\Environment"), "/v", "PATH"
]
expected = gen_expected_regex([
("HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\"
"Control\\\\Session Manager\\\\Environment"), "PATH",
"REG_(EXPAND_)?SZ", "(.*)"
])
p = Popen(cmd, stdout=PIPE, stderr=PIPE,
shell=True, text=True)
out_, _ = p.communicate()
out_ = out_.strip()
if p.returncode == 0:
match = re.match(expected, out_)
if match:
paths.extend(match.group(2).split(os.pathsep))
cmd = ["REG", "QUERY", "HKCU\\Environment", "/v", "PATH"]
expected = gen_expected_regex([
"HKEY_CURRENT_USER\\\\Environment", "PATH", "REG_(EXPAND_)?SZ",
"(.*)"
])
p = Popen(cmd, stdout=PIPE, stderr=PIPE,
shell=True, text=True)
out_, _ = p.communicate()
out_ = out_.strip()
if p.returncode == 0:
match = re.match(expected, out_)
if match:
paths.extend(match.group(2).split(os.pathsep))
cls.syspaths = [x for x in paths if x]
return cls.syspaths
def _open_url(cls, url):
if config.browser:
cmd = [config.browser, url]
if not config.quiet:
print("running command: %s" % " ".join(cmd))
p = Popen(cmd)
p.communicate()
else:
if not config.quiet:
print("opening URL in browser: %s" % url)
webbrowser.open_new(url)
def _physical_cores_from_wmic(self):
# windows
import subprocess
try:
p = Popen('wmic cpu get NumberOfCores /value'.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
except (OSError, IOError):
return None
stdout, stderr = p.communicate()
if p.returncode:
return None
# a Windows machine with 1 installed CPU will return "NumberOfCores=N" where N is
# the number of physical cores on that CPU chip. If more than one CPU is installed
# there will be one "NumberOfCores=N" line listed per actual CPU, so the sum of all
# N is the number of physical cores in the machine: this will be exactly one half the
# number of logical cores (ie from multiprocessing.cpu_count) if HyperThreading is
# enabled on the CPU(s)
result = re.findall(r'NumberOfCores=(\d+)', stdout.strip())
if not result:
# don't know what's wrong... should get back a result like:
# NumberOfCores=2
return None
return sum(map(int, result))
def _physical_cores_from_lscpu(self):
import subprocess
try:
p = Popen(['lscpu'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
except (OSError, IOError):
return None
stdout, stderr = p.communicate()
if p.returncode:
return None
data = self._parse_colon_table_to_dict(stdout)
# lscpu gives output like this:
#
# CPU(s): 24
# On-line CPU(s) list: 0-23
# Thread(s) per core: 2
# Core(s) per socket: 6
# Socket(s): 2
# we want to take sockets * cores, and ignore threads...
# some versions of lscpu format the sockets line differently...
sockets = data.get('Socket(s)', data.get('CPU socket(s)'))
if not sockets:
return None
cores = data.get('Core(s) per socket')
if not cores:
return None
return int(sockets) * int(cores)
def find_site_python(module_name, paths=None):
"""Find the rez native python package that contains the given module.
This function is used by python 'native' rez installers to find the native
rez python package that represents the python installation that this module
is installed into.
Note:
This function is dependent on the behavior found in the python '_native'
package found in the 'rez-recipes' repository. Specifically, it expects
to find a python package with a '_site_paths' list attribute listing
the site directories associated with the python installation.
Args:
module_name (str): Target python module.
paths (list of str, optional): paths to search for packages,
defaults to `config.packages_path`.
Returns:
`Package`: Native python package containing the named module.
"""
from rez.packages import iter_packages
import subprocess
import ast
import os
py_cmd = 'import {x}; print({x}.__path__)'.format(x=module_name)
p = Popen(["python", "-c", py_cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
out, err = p.communicate()
if p.returncode:
raise InvalidPackageError(
"Failed to find installed python module '%s':\n%s" %
(module_name, err))
module_paths = ast.literal_eval(out.strip())
def issubdir(path, parent_path):
return path.startswith(parent_path + os.sep)
for package in iter_packages("python", paths=paths):
if not hasattr(package, "_site_paths"):
continue
contained = True
for module_path in module_paths:
if not any(issubdir(module_path, x) for x in package._site_paths):
contained = False
if contained:
return package
raise InvalidPackageError(
"Failed to find python installation containing the module '%s'. Has "
"python been installed as a rez package?" % module_name)
def exec_python(attr, src, executable="python"):
"""Runs a python subproc to calculate a package attribute.
Args:
attr (str): Name of package attribute being created.
src (list of str): Python code to execute, will be converted into
semicolon-delimited single line of code.
Returns:
str: Output of python process.
"""
import subprocess
if isinstance(src, basestring):
src = [src]
p = Popen([executable, "-c", "; ".join(src)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
out, err = p.communicate()
if p.returncode:
from rez.exceptions import InvalidPackageError
raise InvalidPackageError(
"Error determining package attribute '%s':\n%s" % (attr, err))
return out.strip()
def _cmd(self, *nargs):
"""Convenience function for executing a program such as 'git' etc."""
cmd_str = ' '.join(map(quote, nargs))
if self.package.config.debug("package_release"):
print_debug("Running command: %s" % cmd_str)
p = Popen(nargs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.pkg_root,
text=True)
out, err = p.communicate()
if p.returncode:
print_debug("command stdout:")
print_debug(out)
print_debug("command stderr:")
print_debug(err)
raise ReleaseVCSError("command failed: %s\n%s" % (cmd_str, err))
out = out.strip()
if out:
return [x.rstrip() for x in out.split('\n')]
else:
return []
def get_syspaths(cls):
if cls.syspaths is not None:
return cls.syspaths
if config.standard_system_paths:
cls.syspaths = config.standard_system_paths
return cls.syspaths
# detect system paths using registry
cmd = "cmd=`which %s`; unset PATH; $cmd %s %s 'echo __PATHS_ $PATH'" \
% (cls.name(), cls.norc_arg, cls.command_arg)
p = Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True, text=True)
out_, err_ = p.communicate()
if p.returncode:
paths = []
else:
lines = out_.split('\n')
line = [x for x in lines if "__PATHS_" in x.split()][0]
paths = line.strip().split()[-1].split(os.pathsep)
for path in os.defpath.split(os.path.pathsep):
if path not in paths:
paths.append(path)
cls.syspaths = [x for x in paths if x]
return cls.syspaths
def exec_command(attr, cmd):
"""Runs a subproc to calculate a package attribute.
"""
import subprocess
p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
out, err = p.communicate()
if p.returncode:
from rez.exceptions import InvalidPackageError
raise InvalidPackageError(
"Error determining package attribute '%s':\n%s" % (attr, err))
return out.strip(), err.strip()
def _physical_cores_from_osx_sysctl(self):
import subprocess
try:
p = Popen(['sysctl', '-n', 'hw.physicalcpu'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
except (OSError, IOError):
return None
stdout, stderr = p.communicate()
if p.returncode:
return None
return int(stdout.strip())
def _run_command(args):
cmd_str = ' '.join(quote(x) for x in args)
log("running: %s" % cmd_str)
# https://github.com/nerdvegas/rez/pull/659
use_shell = ("Windows" in platform.system())
p = Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=use_shell,
text=True)
stdout, stderr = p.communicate()
return stdout, stderr, p.returncode
def get_system_info():
"""Get system info that might affect resolve time.
"""
from rez import __version__
from rez.utils.execution import Popen
from rez.solver import SOLVER_VERSION
info = {
"rez_version": __version__,
"rez_solver_version": SOLVER_VERSION,
"py_version": "%d.%d" % sys.version_info[:2],
"platform": platform.platform()
}
# this may only work on linux, but that's ok - the important thing is that
# it works in the benchmark workflow, and we run that on linux only
#
try:
proc = Popen(["cat", "/proc/cpuinfo"],
stdout=subprocess.PIPE,
text=True)
out, _ = proc.communicate()
if proc.returncode == 0:
# parse output, lines are like 'field : value'
fields = {}
for line in out.strip().split('\n'):
if ':' not in line:
continue
parts = line.strip().split(':', 1)
key = parts[0].strip()
value = parts[1].strip()
fields[key] = value
# get the bits we care about
info["num_cpu"] = int(fields["processor"]) + 1
info["cpu"] = fields["model name"]
except:
pass
return info
def _os(self):
"""
Note: We cannot replace this with 'distro.linux_distribution' in
entirety as unfortunately there are slight differences. Eg our code
gives 'Ubuntu-16.04' whereas distro gives 'ubuntu-16.04'.
"""
distributor = None
release = None
def _str(s):
if (s.startswith("'") and s.endswith("'")) \
or (s.startswith('"') and s.endswith('"')):
return s[1:-1]
else:
return s
def _os():
if distributor and release:
return "%s-%s" % (distributor, release)
else:
return None
def _parse(txt, distributor_key, release_key):
distributor_ = None
release_ = None
lines = txt.strip().split('\n')
for line in lines:
if line.startswith(distributor_key):
s = line[len(distributor_key):].strip()
distributor_ = _str(s)
elif line.startswith(release_key):
s = line[len(release_key):].strip()
release_ = _str(s)
return distributor_, release_
# first try parsing the /etc/lsb-release file
file = "/etc/lsb-release"
if os.path.isfile(file):
with open(file) as f:
txt = f.read()
distributor, release = _parse(txt, "DISTRIB_ID=",
"DISTRIB_RELEASE=")
result = _os()
if result:
return result
# next, try getting the output of the lsb_release program
import subprocess
p = Popen(['/usr/bin/env', 'lsb_release', '-a'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
txt = p.communicate()[0]
if not p.returncode:
distributor_, release_ = _parse(txt, "Distributor ID:", "Release:")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# try to read the /etc/os-release file
# this file contains OS specific data on linux
# distributions
# see https://www.freedesktop.org/software/systemd/man/os-release.html
os_release = '/etc/os-release'
if os.path.isfile(os_release):
with open(os_release, 'r') as f:
txt = f.read()
distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# use distro lib
from rez.vendor.distro import distro
parts = distro.linux_distribution(full_distribution_name=False)
if parts[0] == '':
raise RezSystemError("cannot detect operating system")
return '-'.join(parts[:2])
def _os(self):
distributor = None
release = None
def _str(s):
if (s.startswith("'") and s.endswith("'")) \
or (s.startswith('"') and s.endswith('"')):
return s[1:-1]
else:
return s
def _os():
if distributor and release:
return "%s-%s" % (distributor, release)
else:
return None
def _parse(txt, distributor_key, release_key):
distributor_ = None
release_ = None
lines = txt.strip().split('\n')
for line in lines:
if line.startswith(distributor_key):
s = line[len(distributor_key):].strip()
distributor_ = _str(s)
elif line.startswith(release_key):
s = line[len(release_key):].strip()
release_ = _str(s)
return distributor_, release_
# first try parsing the /etc/lsb-release file
file = "/etc/lsb-release"
if os.path.isfile(file):
with open(file) as f:
txt = f.read()
distributor, release = _parse(txt, "DISTRIB_ID=",
"DISTRIB_RELEASE=")
result = _os()
if result:
return result
# next, try getting the output of the lsb_release program
import subprocess
p = Popen(['/usr/bin/env', 'lsb_release', '-a'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
txt = p.communicate()[0]
if not p.returncode:
distributor_, release_ = _parse(txt, "Distributor ID:", "Release:")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# try to read the /etc/os-release file
# this file contains OS specific data on linux
# distributions
# see https://www.freedesktop.org/software/systemd/man/os-release.html
os_release = '/etc/os-release'
if os.path.isfile(os_release):
with open(os_release, 'r') as f:
txt = f.read()
distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=")
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# last, use python's dist detection. It is known to return incorrect
# info on some systems though
try:
distributor_, release_, _ = platform.linux_distribution()
except:
distributor_, release_, _ = platform.dist()
if distributor_ and not distributor:
distributor = distributor_
if release_ and not release:
release = release_
result = _os()
if result:
return result
# last resort, accept missing release
if distributor:
return distributor
# give up
raise RezSystemError("cannot detect operating system")
