def get_syspaths(cls): if cls.syspaths is not None: return cls.syspaths if config.standard_system_paths: cls.syspaths = config.standard_system_paths return cls.syspaths # detect system paths using registry def gen_expected_regex(parts): whitespace = r"[\s]+" return whitespace.join(parts) # TODO: Research if there is an easier way to pull system PATH from # registry in powershell paths = [] cmd = [ "REG", "QUERY", ("HKLM\\SYSTEM\\CurrentControlSet\\" "Control\\Session Manager\\Environment"), "/v", "PATH" ] expected = gen_expected_regex([ ("HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\" "Control\\\\Session Manager\\\\Environment"), "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cmd = ["REG", "QUERY", "HKCU\\Environment", "/v", "PATH"] expected = gen_expected_regex([ "HKEY_CURRENT_USER\\\\Environment", "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, text=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cls.syspaths = [x for x in paths if x] return cls.syspaths
def _open_url(cls, url): if config.browser: cmd = [config.browser, url] if not config.quiet: print("running command: %s" % " ".join(cmd)) p = Popen(cmd) p.communicate() else: if not config.quiet: print("opening URL in browser: %s" % url) webbrowser.open_new(url)
def _physical_cores_from_wmic(self): # windows import subprocess try: p = Popen('wmic cpu get NumberOfCores /value'.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) except (OSError, IOError): return None stdout, stderr = p.communicate() if p.returncode: return None # a Windows machine with 1 installed CPU will return "NumberOfCores=N" where N is # the number of physical cores on that CPU chip. If more than one CPU is installed # there will be one "NumberOfCores=N" line listed per actual CPU, so the sum of all # N is the number of physical cores in the machine: this will be exactly one half the # number of logical cores (ie from multiprocessing.cpu_count) if HyperThreading is # enabled on the CPU(s) result = re.findall(r'NumberOfCores=(\d+)', stdout.strip()) if not result: # don't know what's wrong... should get back a result like: # NumberOfCores=2 return None return sum(map(int, result))
def _physical_cores_from_lscpu(self): import subprocess try: p = Popen(['lscpu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) except (OSError, IOError): return None stdout, stderr = p.communicate() if p.returncode: return None data = self._parse_colon_table_to_dict(stdout) # lscpu gives output like this: # # CPU(s): 24 # On-line CPU(s) list: 0-23 # Thread(s) per core: 2 # Core(s) per socket: 6 # Socket(s): 2 # we want to take sockets * cores, and ignore threads... # some versions of lscpu format the sockets line differently... sockets = data.get('Socket(s)', data.get('CPU socket(s)')) if not sockets: return None cores = data.get('Core(s) per socket') if not cores: return None return int(sockets) * int(cores)
def find_site_python(module_name, paths=None): """Find the rez native python package that contains the given module. This function is used by python 'native' rez installers to find the native rez python package that represents the python installation that this module is installed into. Note: This function is dependent on the behavior found in the python '_native' package found in the 'rez-recipes' repository. Specifically, it expects to find a python package with a '_site_paths' list attribute listing the site directories associated with the python installation. Args: module_name (str): Target python module. paths (list of str, optional): paths to search for packages, defaults to `config.packages_path`. Returns: `Package`: Native python package containing the named module. """ from rez.packages import iter_packages import subprocess import ast import os py_cmd = 'import {x}; print({x}.__path__)'.format(x=module_name) p = Popen(["python", "-c", py_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() if p.returncode: raise InvalidPackageError( "Failed to find installed python module '%s':\n%s" % (module_name, err)) module_paths = ast.literal_eval(out.strip()) def issubdir(path, parent_path): return path.startswith(parent_path + os.sep) for package in iter_packages("python", paths=paths): if not hasattr(package, "_site_paths"): continue contained = True for module_path in module_paths: if not any(issubdir(module_path, x) for x in package._site_paths): contained = False if contained: return package raise InvalidPackageError( "Failed to find python installation containing the module '%s'. Has " "python been installed as a rez package?" % module_name)
def exec_python(attr, src, executable="python"): """Runs a python subproc to calculate a package attribute. Args: attr (str): Name of package attribute being created. src (list of str): Python code to execute, will be converted into semicolon-delimited single line of code. Returns: str: Output of python process. """ import subprocess if isinstance(src, basestring): src = [src] p = Popen([executable, "-c", "; ".join(src)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() if p.returncode: from rez.exceptions import InvalidPackageError raise InvalidPackageError( "Error determining package attribute '%s':\n%s" % (attr, err)) return out.strip()
def _cmd(self, *nargs): """Convenience function for executing a program such as 'git' etc.""" cmd_str = ' '.join(map(quote, nargs)) if self.package.config.debug("package_release"): print_debug("Running command: %s" % cmd_str) p = Popen(nargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.pkg_root, text=True) out, err = p.communicate() if p.returncode: print_debug("command stdout:") print_debug(out) print_debug("command stderr:") print_debug(err) raise ReleaseVCSError("command failed: %s\n%s" % (cmd_str, err)) out = out.strip() if out: return [x.rstrip() for x in out.split('\n')] else: return []
def get_syspaths(cls): if cls.syspaths is not None: return cls.syspaths if config.standard_system_paths: cls.syspaths = config.standard_system_paths return cls.syspaths # detect system paths using registry cmd = "cmd=`which %s`; unset PATH; $cmd %s %s 'echo __PATHS_ $PATH'" \ % (cls.name(), cls.norc_arg, cls.command_arg) p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) out_, err_ = p.communicate() if p.returncode: paths = [] else: lines = out_.split('\n') line = [x for x in lines if "__PATHS_" in x.split()][0] paths = line.strip().split()[-1].split(os.pathsep) for path in os.defpath.split(os.path.pathsep): if path not in paths: paths.append(path) cls.syspaths = [x for x in paths if x] return cls.syspaths
def exec_command(attr, cmd): """Runs a subproc to calculate a package attribute. """ import subprocess p = Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() if p.returncode: from rez.exceptions import InvalidPackageError raise InvalidPackageError( "Error determining package attribute '%s':\n%s" % (attr, err)) return out.strip(), err.strip()
def _physical_cores_from_osx_sysctl(self): import subprocess try: p = Popen(['sysctl', '-n', 'hw.physicalcpu'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) except (OSError, IOError): return None stdout, stderr = p.communicate() if p.returncode: return None return int(stdout.strip())
def _run_command(args): cmd_str = ' '.join(quote(x) for x in args) log("running: %s" % cmd_str) # https://github.com/nerdvegas/rez/pull/659 use_shell = ("Windows" in platform.system()) p = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=use_shell, text=True) stdout, stderr = p.communicate() return stdout, stderr, p.returncode
def get_system_info(): """Get system info that might affect resolve time. """ from rez import __version__ from rez.utils.execution import Popen from rez.solver import SOLVER_VERSION info = { "rez_version": __version__, "rez_solver_version": SOLVER_VERSION, "py_version": "%d.%d" % sys.version_info[:2], "platform": platform.platform() } # this may only work on linux, but that's ok - the important thing is that # it works in the benchmark workflow, and we run that on linux only # try: proc = Popen(["cat", "/proc/cpuinfo"], stdout=subprocess.PIPE, text=True) out, _ = proc.communicate() if proc.returncode == 0: # parse output, lines are like 'field : value' fields = {} for line in out.strip().split('\n'): if ':' not in line: continue parts = line.strip().split(':', 1) key = parts[0].strip() value = parts[1].strip() fields[key] = value # get the bits we care about info["num_cpu"] = int(fields["processor"]) + 1 info["cpu"] = fields["model name"] except: pass return info
def _os(self): """ Note: We cannot replace this with 'distro.linux_distribution' in entirety as unfortunately there are slight differences. Eg our code gives 'Ubuntu-16.04' whereas distro gives 'ubuntu-16.04'. """ distributor = None release = None def _str(s): if (s.startswith("'") and s.endswith("'")) \ or (s.startswith('"') and s.endswith('"')): return s[1:-1] else: return s def _os(): if distributor and release: return "%s-%s" % (distributor, release) else: return None def _parse(txt, distributor_key, release_key): distributor_ = None release_ = None lines = txt.strip().split('\n') for line in lines: if line.startswith(distributor_key): s = line[len(distributor_key):].strip() distributor_ = _str(s) elif line.startswith(release_key): s = line[len(release_key):].strip() release_ = _str(s) return distributor_, release_ # first try parsing the /etc/lsb-release file file = "/etc/lsb-release" if os.path.isfile(file): with open(file) as f: txt = f.read() distributor, release = _parse(txt, "DISTRIB_ID=", "DISTRIB_RELEASE=") result = _os() if result: return result # next, try getting the output of the lsb_release program import subprocess p = Popen(['/usr/bin/env', 'lsb_release', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) txt = p.communicate()[0] if not p.returncode: distributor_, release_ = _parse(txt, "Distributor ID:", "Release:") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # try to read the /etc/os-release file # this file contains OS specific data on linux # distributions # see https://www.freedesktop.org/software/systemd/man/os-release.html os_release = '/etc/os-release' if os.path.isfile(os_release): with open(os_release, 'r') as f: txt = f.read() distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # use distro lib from rez.vendor.distro import distro parts = distro.linux_distribution(full_distribution_name=False) if parts[0] == '': raise RezSystemError("cannot detect operating system") return '-'.join(parts[:2])
def _os(self): distributor = None release = None def _str(s): if (s.startswith("'") and s.endswith("'")) \ or (s.startswith('"') and s.endswith('"')): return s[1:-1] else: return s def _os(): if distributor and release: return "%s-%s" % (distributor, release) else: return None def _parse(txt, distributor_key, release_key): distributor_ = None release_ = None lines = txt.strip().split('\n') for line in lines: if line.startswith(distributor_key): s = line[len(distributor_key):].strip() distributor_ = _str(s) elif line.startswith(release_key): s = line[len(release_key):].strip() release_ = _str(s) return distributor_, release_ # first try parsing the /etc/lsb-release file file = "/etc/lsb-release" if os.path.isfile(file): with open(file) as f: txt = f.read() distributor, release = _parse(txt, "DISTRIB_ID=", "DISTRIB_RELEASE=") result = _os() if result: return result # next, try getting the output of the lsb_release program import subprocess p = Popen(['/usr/bin/env', 'lsb_release', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) txt = p.communicate()[0] if not p.returncode: distributor_, release_ = _parse(txt, "Distributor ID:", "Release:") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # try to read the /etc/os-release file # this file contains OS specific data on linux # distributions # see https://www.freedesktop.org/software/systemd/man/os-release.html os_release = '/etc/os-release' if os.path.isfile(os_release): with open(os_release, 'r') as f: txt = f.read() distributor_, release_ = _parse(txt, "ID=", "VERSION_ID=") if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # last, use python's dist detection. It is known to return incorrect # info on some systems though try: distributor_, release_, _ = platform.linux_distribution() except: distributor_, release_, _ = platform.dist() if distributor_ and not distributor: distributor = distributor_ if release_ and not release: release = release_ result = _os() if result: return result # last resort, accept missing release if distributor: return distributor # give up raise RezSystemError("cannot detect operating system")