def get_py_venv_executable(dest_dir): # get virtualenv's python executable _, _, _, venv_bin_dir = path_locations(dest_dir) env = {"PATH": venv_bin_dir, "PATHEXT": os.environ.get("PATHEXT", "")} return venv_bin_dir, which("python", env=env)
def get_syspaths(cls): if cls.syspaths is not None: return cls.syspaths if config.standard_system_paths: cls.syspaths = config.standard_system_paths return cls.syspaths # detect system paths using registry cmd = "cmd=`which %s`; unset PATH; $cmd %s %s 'echo __PATHS_ $PATH'" \ % (cls.name(), cls.norc_arg, cls.command_arg) p = popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out_, err_ = p.communicate() if p.returncode: paths = [] else: lines = out_.split('\n') line = [x for x in lines if "__PATHS_" in x.split()][0] paths = line.strip().split()[-1].split(os.pathsep) for path in os.defpath.split(os.path.pathsep): if path not in paths: paths.append(path) cls.syspaths = [x for x in paths if x] # add Rez binaries exe = which("rez-env") assert exe, "Could not find rez binary, this is a bug" rez_bin_dir = os.path.dirname(exe) cls.syspaths.insert(0, rez_bin_dir) return cls.syspaths
def get_virtualenv_py_executable(dest_dir): # get virtualenv's python executable bin_dir = get_virtualenv_bin_dir(dest_dir) env = {"PATH": bin_dir, "PATHEXT": os.environ.get("PATHEXT", "")} return bin_dir, which("python", env=env)
def patch_rez_binaries(dest_dir): bin_names = os.listdir(bin_path) _, _, _, venv_bin_path = path_locations(dest_dir) venv_py_executable = which("python", env={ "PATH": venv_bin_path, "PATHEXT": os.environ.get("PATHEXT", "") }) # delete rez bin files written by setuptools for name in bin_names: filepath = os.path.join(venv_bin_path, name) if os.path.isfile(filepath): os.remove(filepath) # write patched bins instead. These go into 'bin/rez' subdirectory, which # gives us a bin dir containing only rez binaries. This is what we want - # we don't want resolved envs accidentally getting the venv's 'python'. dest_bin_path = os.path.join(venv_bin_path, "rez") if os.path.exists(dest_bin_path): shutil.rmtree(dest_bin_path) os.makedirs(dest_bin_path) maker = _ScriptMaker(bin_path, dest_bin_path) maker.executable = venv_py_executable options = dict(interpreter_args=["-E"]) for name in bin_names: entry = fake_entry(name) maker._make_script(entry, [], options=options)
def patch_rez_binaries(dest_dir): bin_names = os.listdir(bin_path) _, _, _, venv_bin_path = path_locations(dest_dir) venv_py_executable = which("python", env={"PATH":venv_bin_path, "PATHEXT":os.environ.get("PATHEXT", "")}) # delete rez bin files written by setuptools for name in bin_names: filepath = os.path.join(venv_bin_path, name) if os.path.isfile(filepath): os.remove(filepath) # write patched bins instead. These go into 'bin/rez' subdirectory, which # gives us a bin dir containing only rez binaries. This is what we want - # we don't want resolved envs accidentally getting the venv's 'python'. dest_bin_path = os.path.join(venv_bin_path, "rez") if os.path.exists(dest_bin_path): shutil.rmtree(dest_bin_path) os.makedirs(dest_bin_path) maker = _ScriptMaker(bin_path, dest_bin_path) maker.executable = venv_py_executable options = dict(interpreter_args=["-E"]) for name in bin_names: entry = fake_entry(name) maker._make_script(entry, [], options=options)
def find_python(python_version=None): """Find a pip exe using the given python version. Returns: 2-tuple: str: pip executable; `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ python_exe = "python" try: context = create_context(python_version) except BuildError as e: # fall back on system pip. Not ideal but at least it's something from rez.backport.shutilwhich import which python_exe = which("python") if python_exe: print_warning( "python rez package could not be found; system 'python' " "command (%s) will be used instead." % python_exe) context = None else: raise e return python_exe, context
def find_pip(pip_version=None, python_version=None): """Find a pip exe using the given python version. Returns: 2-tuple: str: pip executable; `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ pip_exe = "pip" try: context = create_context(pip_version, python_version) except BuildError as e: # fall back on system pip. Not ideal but at least it's something from rez.backport.shutilwhich import which pip_exe = which("pip") if pip_exe: print_warning( "pip rez package could not be found; system 'pip' command (%s) " "will be used instead." % pip_exe) context = None else: raise e return pip_exe, context
def find_pip(pip_version=None, python_version=None): """Find a pip exe using the given python version. Returns: 2-tuple: str: pip executable; `ResolvedContext`: Context containing pip, or None if we fell back to system pip. """ pip_exe = "pip" try: context = create_context(pip_version, python_version) except BuildError: # fall back on system pip. Not ideal but at least it's something from rez.backport.shutilwhich import which pip_exe = which("pip") if pip_exe: print_warning( "pip rez package could not be found; system 'pip' command (%s) " "will be used instead." % pip_exe) context = None else: raise # check pip version, must be >=19 to support PEP517 try: pattern = r"pip\s(?P<ver>\d+\.*\d*\.*\d*)" if "Windows" in platform.system(): # https://github.com/nerdvegas/rez/pull/659 ver_str = subprocess.check_output(pip_exe + " -V", shell=True, universal_newlines=True) else: ver_str = subprocess.check_output([pip_exe, '-V'], universal_newlines=True) match = re.search(pattern, ver_str) ver = match.group('ver') pip_major = ver.split('.')[0] if int(pip_major) < 19: raise VersionError( "pip >= 19 is required! Please update your pip.") except VersionError: raise except: # silently skip if pip version detection failed, pip itself will show # a reasonable error message at the least. pass return pip_exe, context
def find_executable(cls, name, check_syspaths=False): """Find an executable. Args: name (str): Program name. check_syspaths (bool): If True, check the standard system paths as well, if program was not found on current $PATH. Returns: str: Full filepath of executable. """ exe = which(name) if not exe and check_syspaths: paths = cls.get_syspaths() env = os.environ.copy() env["PATH"] = os.pathsep.join(paths) exe = which(name, env=env) if not exe: raise RuntimeError("Couldn't find executable '%s'." % name) return exe
def build(self, context, variant, build_path, install_path, install=False, build_type=BuildType.local): def _pr(s): if self.verbose: print s # find cmake binary if self.settings.cmake_binary: exe = self.settings.cmake_binary else: exe = context.which("cmake", fallback=True) if not exe: raise RezCMakeError("could not find cmake binary") found_exe = which(exe) if not found_exe: raise RezCMakeError("cmake binary does not exist: %s" % exe) sh = create_shell() # assemble cmake command cmd = [found_exe, "-d", self.working_dir] cmd += (self.settings.cmake_args or []) cmd += (self.build_args or []) cmd.append("-DCMAKE_INSTALL_PREFIX=%s" % install_path) cmd.append("-DCMAKE_MODULE_PATH=%s" % sh.get_key_token("CMAKE_MODULE_PATH")) cmd.append("-DCMAKE_BUILD_TYPE=%s" % self.build_target) cmd.append("-DREZ_BUILD_TYPE=%s" % build_type.name) cmd.append("-DREZ_BUILD_INSTALL=%d" % (1 if install else 0)) cmd.extend(["-G", self.build_systems[self.cmake_build_system]]) if config.rez_1_cmake_variables and \ not config.disable_rez_1_compatibility and \ build_type == BuildType.central: cmd.append("-DCENTRAL=1") # execute cmake within the build env _pr("Executing: %s" % ' '.join(cmd)) if not os.path.abspath(build_path): build_path = os.path.join(self.working_dir, build_path) build_path = os.path.realpath(build_path) callback = functools.partial(self._add_build_actions, context=context, package=self.package, variant=variant, build_type=build_type) # run the build command and capture/print stderr at the same time retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) ret = {} if retcode: ret["success"] = False return ret if self.write_build_scripts: # write out the script that places the user in a build env, where # they can run make directly themselves. build_env_script = os.path.join(build_path, "build-env") create_forwarding_script(build_env_script, module=("build_system", "cmake"), func_name="_FWD__spawn_build_shell", working_dir=self.working_dir, build_dir=build_path, variant_index=variant.index) ret["success"] = True ret["build_env_script"] = build_env_script return ret # assemble make command if self.settings.make_binary: cmd = [self.settings.make_binary] else: cmd = ["make"] cmd += (self.child_build_args or []) if not any(x.startswith("-j") for x in (self.child_build_args or [])): n = variant.config.build_thread_count or cpu_count() cmd.append("-j%d" % n) # execute make within the build env _pr("\nExecuting: %s" % ' '.join(cmd)) retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) if not retcode and install and "install" not in cmd: cmd.append("install") # execute make install within the build env _pr("\nExecuting: %s" % ' '.join(cmd)) retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) ret["success"] = (not retcode) return ret
def print_tools(self, pattern=None, buf=sys.stdout): """Print a list of visible tools. Args: pattern (str): Only list tools that match this glob pattern. """ seen = set() rows = [] context = self.context if context: data = context.get_tools() conflicts = set(context.get_conflicting_tools().keys()) for _, (variant, tools) in sorted(data.items()): pkg_str = variant.qualified_package_name for tool in tools: if pattern and not fnmatch(tool, pattern): continue if tool in conflicts: label = "(in conflict)" color = critical else: label = '' color = None rows.append([tool, '-', pkg_str, "active context", label, color]) seen.add(tool) for suite in self.suites: for tool, d in suite.get_tools().iteritems(): if tool in seen: continue if pattern and not fnmatch(tool, pattern): continue label = [] color = None path = which(tool) if path: path_ = os.path.join(suite.tools_path, tool) if path != path_: label.append("(hidden by unknown tool '%s')" % path) color = warning variant = d["variant"] if isinstance(variant, set): pkg_str = ", ".join(variant) label.append("(in conflict)") color = critical else: pkg_str = variant.qualified_package_name orig_tool = d["tool_name"] if orig_tool == tool: orig_tool = '-' label = ' '.join(label) source = ("context '%s' in suite '%s'" % (d["context_name"], suite.load_path)) rows.append([tool, orig_tool, pkg_str, source, label, color]) seen.add(tool) _pr = Printer(buf) if not rows: _pr("No matching tools.") return False headers = [["TOOL", "ALIASING", "PACKAGE", "SOURCE", "", None], ["----", "--------", "-------", "------", "", None]] rows = headers + sorted(rows, key=lambda x: x[0].lower()) print_colored_columns(_pr, rows) return True
dest_dir = os.path.expanduser(dest_dir) dest_dir = os.path.realpath(dest_dir) print "installing rez to %s..." % dest_dir # make virtualenv verbose log_level = Logger.level_for_integer(2 - opts.verbose) logger = Logger([(log_level, sys.stdout)]) # create the virtualenv create_environment(dest_dir) # install rez from source _, _, _, venv_bin_dir = path_locations(dest_dir) py_executable = which("python", env={"PATH":venv_bin_dir, "PATHEXT":os.environ.get("PATHEXT", "")}) args = [ py_executable, "setup.py", "install", "--quiet", "--single-version-externally-managed", "--record", os.path.join(dest_dir, 'install_log.txt')] if opts.verbose: print "running in %s: %s" % (source_path, " ".join(args)) p = subprocess.Popen(args, cwd=source_path) p.wait() # patch the rez binaries patch_rez_binaries(dest_dir) # copy completion scripts into venv completion_path = copy_completion_scripts(dest_dir)
dest_dir = os.path.realpath(dest_dir) print "installing rez to %s..." % dest_dir # make virtualenv verbose log_level = Logger.level_for_integer(2 - opts.verbose) logger = Logger([(log_level, sys.stdout)]) # create the virtualenv create_environment(dest_dir) # install rez from source _, _, _, venv_bin_dir = path_locations(dest_dir) py_executable = which("python", env={ "PATH": venv_bin_dir, "PATHEXT": os.environ.get("PATHEXT", "") }) args = [ py_executable, "setup.py", "install", "--quiet", "--single-version-externally-managed", "--record", os.path.join(dest_dir, 'install_log.txt') ] if opts.verbose: print "running in %s: %s" % (source_path, " ".join(args)) p = subprocess.Popen(args, cwd=source_path) p.wait() # patch the rez binaries patch_rez_binaries(dest_dir)
def build(self, context, variant, build_path, install_path, install=False, build_type=BuildType.local): def _pr(s): if self.verbose: print s # find cmake binary if self.settings.cmake_binary: exe = self.settings.cmake_binary else: exe = context.which("cmake", fallback=True) if not exe: raise RezCMakeError("could not find cmake binary") found_exe = which(exe) if not found_exe: raise RezCMakeError("cmake binary does not exist: %s" % exe) sh = create_shell() # assemble cmake command cmd = [found_exe, "-d", self.working_dir] cmd += (self.settings.cmake_args or []) cmd += (self.build_args or []) cmd.append("-DCMAKE_INSTALL_PREFIX=%s" % install_path) cmd.append("-DCMAKE_MODULE_PATH=%s" % sh.get_key_token("CMAKE_MODULE_PATH")) cmd.append("-DCMAKE_BUILD_TYPE=%s" % self.build_target) cmd.append("-DREZ_BUILD_TYPE=%s" % build_type.name) cmd.append("-DREZ_BUILD_INSTALL=%d" % (1 if install else 0)) cmd.extend(["-G", self.build_systems[self.cmake_build_system]]) if config.rez_1_cmake_variables and \ not config.disable_rez_1_compatibility and \ build_type == BuildType.central: cmd.append("-DCENTRAL=1") # execute cmake within the build env _pr("Executing: %s" % ' '.join(cmd)) if not os.path.abspath(build_path): build_path = os.path.join(self.working_dir, build_path) build_path = os.path.realpath(build_path) callback = functools.partial(self._add_build_actions, context=context, package=self.package, variant=variant, build_type=build_type) # run the build command and capture/print stderr at the same time retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) ret = {} if retcode: ret["success"] = False return ret if self.write_build_scripts: # write out the script that places the user in a build env, where # they can run make directly themselves. build_env_script = os.path.join(build_path, "build-env") create_forwarding_script(build_env_script, module=("build_system", "cmake"), func_name="_FWD__spawn_build_shell", working_dir=self.working_dir, build_dir=build_path, variant_index=variant.index) ret["success"] = True ret["build_env_script"] = build_env_script return ret # assemble make command if self.settings.make_binary: cmd = [self.settings.make_binary] else: cmd = ["make"] cmd += (self.child_build_args or []) if not any(x.startswith("-j") for x in (self.child_build_args or [])): n = variant.config.build_thread_count cmd.append("-j%d" % n) # execute make within the build env _pr("\nExecuting: %s" % ' '.join(cmd)) retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) if not retcode and install and "install" not in cmd: cmd.append("install") # execute make install within the build env _pr("\nExecuting: %s" % ' '.join(cmd)) retcode, _, _ = context.execute_shell(command=cmd, block=True, cwd=build_path, actions_callback=callback) ret["success"] = (not retcode) return ret
def _print_tool_info(self, value, buf=sys.stdout, b=False): word = "is also" if b else "is" _pr = Printer(buf) def _load_wrapper(filepath): try: return Wrapper(filepath) except: return # find it on disk filepath = None unpathed = (os.path.basename(value) == value) if unpathed: filepath = which(value) if filepath is None: path = os.path.abspath(value) if os.path.exists(path): filepath = path if not filepath or not os.path.isfile(filepath): return False # is it a suite wrapper? tool_name = os.path.basename(filepath) w = _load_wrapper(filepath) if w: _pr("'%s' %s a suite tool:" % (tool_name, word)) w.print_about() return True # is it a tool in a current context? if self.context: variants = self.context.get_tool_variants(tool_name) if variants: _pr("'%s' %s a tool in the active context:" % (tool_name, word)) _pr("Tool: %s" % tool_name) if self.context.load_path: _pr("Context: %s" % self.context.load_path) if len(variants) > 1: vars_str = " ".join(x.qualified_package_name for x in variants) msg = "Packages (in conflict): %s" % vars_str _pr(msg, critical) else: variant = iter(variants).next() _pr("Package: %s" % variant.qualified_package_name) return True # is it actually a suite wrapper, but it's being hidden by another tool # on $PATH with the same name? if unpathed: for suite in self.suites: filepath_ = os.path.join(suite.tools_path, tool_name) if os.path.isfile(filepath_): w = _load_wrapper(filepath_) if w: _pr("'%s' %s a suite tool, but is hidden by an unknown tool '%s':" % (tool_name, word, filepath), warning) w.print_about() return True return False
def _print_tool_info(self, value, buf=sys.stdout, b=False): word = "is also" if b else "is" _pr = Printer(buf) def _load_wrapper(filepath): try: return Wrapper(filepath) except: return # find it on disk filepath = None unpathed = (os.path.basename(value) == value) if unpathed: filepath = which(value) if filepath is None: path = os.path.abspath(value) if os.path.exists(path): filepath = path if not filepath or not os.path.isfile(filepath): return False # is it a suite wrapper? tool_name = os.path.basename(filepath) w = _load_wrapper(filepath) if w: _pr("'%s' %s a suite tool:" % (tool_name, word)) w.print_about() return True # is it a tool in a current context? if self.context: variants = self.context.get_tool_variants(tool_name) if variants: _pr("'%s' %s a tool in the active context:" % (tool_name, word)) _pr("Tool: %s" % tool_name) if self.context.load_path: _pr("Context: %s" % self.context.load_path) if len(variants) > 1: vars_str = " ".join(x.qualified_package_name for x in variants) msg = "Packages (in conflict): %s" % vars_str _pr(msg, critical) else: variant = iter(variants).next() _pr("Package: %s" % variant.qualified_package_name) return True # is it actually a suite wrapper, but it's being hidden by another tool # on $PATH with the same name? if unpathed: for suite in self.suites: filepath_ = os.path.join(suite.tools_path, tool_name) if os.path.isfile(filepath_): w = _load_wrapper(filepath_) if w: _pr( "'%s' %s a suite tool, but is hidden by an unknown tool '%s':" % (tool_name, word, filepath), warning) w.print_about() return True return False
def get_syspaths(cls): if cls.syspaths is not None: return cls.syspaths if config.standard_system_paths: cls.syspaths = config.standard_system_paths return cls.syspaths # detect system paths using registry def gen_expected_regex(parts): whitespace = "[\s]+" return whitespace.join(parts) paths = [] cmd = [ "REG", "QUERY", "HKLM\\SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment", "/v", "PATH" ] expected = gen_expected_regex([ "HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\Control\\\\Session Manager\\\\Environment", "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cmd = [ "REG", "QUERY", "HKCU\\Environment", "/v", "PATH" ] expected = gen_expected_regex([ "HKEY_CURRENT_USER\\\\Environment", "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cls.syspaths = list(set([x for x in paths if x])) # add Rez binaries exe = which("rez-env") assert exe, "Could not find rez binary, this is a bug" rez_bin_dir = os.path.dirname(exe) cls.syspaths.insert(0, rez_bin_dir) return cls.syspaths
def get_syspaths(cls): if cls.syspaths is not None: return cls.syspaths if config.standard_system_paths: cls.syspaths = config.standard_system_paths return cls.syspaths # detect system paths using registry def gen_expected_regex(parts): whitespace = r"[\s]+" return whitespace.join(parts) # TODO: Research if there is an easier way to pull system PATH from # registry in powershell paths = [] cmd = [ "REG", "QUERY", ("HKLM\\SYSTEM\\CurrentControlSet\\" "Control\\Session Manager\\Environment"), "/v", "PATH" ] expected = gen_expected_regex([ ("HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\" "Control\\\\Session Manager\\\\Environment"), "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, shell=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cmd = ["REG", "QUERY", "HKCU\\Environment", "/v", "PATH"] expected = gen_expected_regex([ "HKEY_CURRENT_USER\\\\Environment", "PATH", "REG_(EXPAND_)?SZ", "(.*)" ]) p = popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, shell=True) out_, _ = p.communicate() out_ = out_.strip() if p.returncode == 0: match = re.match(expected, out_) if match: paths.extend(match.group(2).split(os.pathsep)) cls.syspaths = list(set([x for x in paths if x])) # add Rez binaries exe = which("rez-env") assert exe, "Could not find rez binary, this is a bug" rez_bin_dir = os.path.dirname(exe) cls.syspaths.insert(0, rez_bin_dir) return cls.syspaths
def print_tools(self, pattern=None, buf=sys.stdout): """Print a list of visible tools. Args: pattern (str): Only list tools that match this glob pattern. """ seen = set() rows = [] context = self.context if context: data = context.get_tools() conflicts = set(context.get_conflicting_tools().keys()) for _, (variant, tools) in sorted(data.items()): pkg_str = variant.qualified_package_name for tool in tools: if pattern and not fnmatch(tool, pattern): continue if tool in conflicts: label = "(in conflict)" color = critical else: label = '' color = None rows.append( [tool, '-', pkg_str, "active context", label, color]) seen.add(tool) for suite in self.suites: for tool, d in suite.get_tools().iteritems(): if tool in seen: continue if pattern and not fnmatch(tool, pattern): continue label = [] color = None path = which(tool) if path: path_ = os.path.join(suite.tools_path, tool) if path != path_: label.append("(hidden by unknown tool '%s')" % path) color = warning variant = d["variant"] if isinstance(variant, set): pkg_str = ", ".join(variant) label.append("(in conflict)") color = critical else: pkg_str = variant.qualified_package_name orig_tool = d["tool_name"] if orig_tool == tool: orig_tool = '-' label = ' '.join(label) source = ("context '%s' in suite '%s'" % (d["context_name"], suite.load_path)) rows.append([tool, orig_tool, pkg_str, source, label, color]) seen.add(tool) _pr = Printer(buf) if not rows: _pr("No matching tools.") return False headers = [["TOOL", "ALIASING", "PACKAGE", "SOURCE", "", None], ["----", "--------", "-------", "------", "", None]] rows = headers + sorted(rows, key=lambda x: x[0].lower()) print_colored_columns(_pr, rows) return True