def resolved_name_version(self): if not self.folder: if self.version: return self.original, self.version return despecced(self.original) setup_py = os.path.join(self.folder, "setup.py") if not os.path.exists(setup_py): abort("No setup.py in %s" % self.folder) with runez.CurrentFolder(self.folder): # Some setup.py's assume current folder is the one with their setup.py r = runez.run(sys.executable, "setup.py", "--name", dryrun=False, fatal=False, logger=False) package_name = r.output if r.failed or not package_name: abort("Could not determine package name from %s" % setup_py) validate_pypi_name(package_name) r = runez.run(sys.executable, "setup.py", "--version", dryrun=False, fatal=False, logger=False) package_version = r.output if r.failed or not package_version: abort("Could not determine package version from setup.py") return package_name, package_version
def effective_package(self, template): """ :param str template: Template describing how to name delivered files, example: {meta}/{name}-{version} """ folder = os.path.join(self.dist_folder, template.format(name=self.package_spec.dashed, version=self.desired.version)) clean_folder(folder) python = system.target_python(package_spec=self.package_spec) if not python.has_builtin_venv or self.relocatable: venv = "virtualenv==16.7.7" else: venv = "venv" vrun(self.package_spec, venv, folder) bin_folder = os.path.join(folder, "bin") pip = os.path.join(bin_folder, "pip") spec = self.source_folder if self.source_folder else "%s==%s" % (self.package_spec.dashed, self.desired.version) runez.run(pip, "install", "-i", system.SETTINGS.index, "-f", self.build_folder, spec) if self.relocatable: python = system.target_python(package_spec=self.package_spec).executable vrun(self.package_spec, "virtualenv", "--relocatable", "--python=%s" % python, folder) runez.run(os.path.join(bin_folder, "python"), "-mcompileall") self.packaged.append(folder) self.executables = [os.path.join(bin_folder, name) for name in self.entry_points]
def test_run(temp_folder): assert runez.program.added_env_paths(None) is None ls = runez.which("ls") with runez.CaptureOutput(dryrun=True) as logged: assert "Would run: /dev/null" in runez.run("/dev/null", fatal=False) assert "Would run: /dev/null" in logged.pop() assert "Would run:" in runez.run(ls, "--invalid-flag", None, ".") assert "Would run: %s ." % ls in logged.pop() with runez.CaptureOutput() as logged: assert runez.run("/dev/null", fatal=False) is False assert "/dev/null is not installed" in logged.pop() assert runez.touch("sample") == 1 assert runez.run(ls, "--invalid-flag", None, ".", path_env={"PATH": ":."}) == "sample" assert "Running: %s ." % ls in logged.pop() assert runez.run(ls, "some-file", fatal=False) is False assert "Running: %s some-file" % ls in logged assert "exited with code" in logged assert "No such file" in logged.pop()
def effective_package(self, template): """ :param str template: Template describing how to name delivered files, example: {meta}/{name}-{version} """ folder = os.path.join( self.dist_folder, template.format(name=self.name, version=self.version)) runez.delete(folder, logger=None) runez.ensure_folder(folder, folder=True, logger=None) vrun(self.name, "virtualenv", folder) bin_folder = os.path.join(folder, "bin") pip = os.path.join(bin_folder, "pip") spec = self.source_folder if self.source_folder else "%s==%s" % ( self.name, self.version) runez.run(pip, "install", "-i", system.SETTINGS.index, "-f", self.build_folder, spec) if self.relocatable: python = system.target_python(package_name=self.name).executable vrun(self.name, "virtualenv", "--relocatable", "--python=%s" % python, folder) self.packaged.append(folder) self.executables = [ os.path.join(bin_folder, name) for name in self.entry_points ]
def upgrade(self): self.stop() with runez.CurrentFolder(self.target): runez.run("docker-compose", "pull") runez.run("docker-compose", "prune", "-f") self.sync() self.start()
def git_clone(pspec): basename = runez.basename(pspec.original) pspec.folder = pspec.cfg.cache.full_path("checkout", basename) runez.ensure_folder(pspec.folder, clean=True, dryrun=False) logger = LOG.info if runez.DRYRUN else runez.UNSET runez.run("git", "clone", pspec.original, pspec.folder, dryrun=False, logger=logger)
def test_capture(temp_folder, logged): chatter = runez.resolved_path("chatter") assert runez.write(chatter, CHATTER.strip(), fatal=False) == 1 assert runez.make_executable(chatter, fatal=False) == 1 assert runez.run(chatter, fatal=False) == "chatter" r = runez.run(chatter, include_error=True, fatal=False) assert r.startswith("chatter") assert "No such file" in r assert "Running: chatter" in logged
def stop_profiler(cls): cls.profiler.disable() filepath = runez.DEV.project_path(".tox", "lastrun.profile") try: cls.profiler.dump_stats(filepath) if runez.which("qcachegrind") is None: print("run 'brew install qcachegrind'") return runez.run("pyprof2calltree", "-k", "-i", filepath, stdout=None, stderr=None) except Exception as e: print("Can't save %s: %s" % (filepath, e))
def check_wrap(self, wrap_method): impl = DeliveryMethod.delivery_method_by_name(wrap_method) impl.install(self.pspec, self.venv, self.entry_points) exe = runez.resolved_path(self.name) r = runez.run(exe, "--version", fatal=False) assert r.succeeded assert r.output == self.version
def request_get(url): """ :param str url: URL to query :return str: Response body """ try: LOG.debug("GET %s", url) request = Request(url) # nosec response = urlopen(request).read() # nosec return response and runez.decode(response).strip() except Exception as e: code = getattr(e, "code", None) if isinstance(code, int) and 400 <= code < 500: return None try: # Some old python installations have trouble with SSL (OSX for example), try curl result = runez.run("curl", "-s", url, dryrun=False, fatal=False) if result.succeeded and result.output: return result.output except Exception as e: LOG.debug("GET %s failed: %s", url, e, exc_info=e) return None
def test_background_run(logged): with runez.CurrentFolder(os.path.dirname(CHATTER)): r = runez.run(CHATTER, "hello", background=True, dryrun=True, logger=True) assert r.succeeded assert "chatter hello &" in logged.pop() r = runez.run(CHATTER, "hello", background=True, dryrun=False) assert r.succeeded assert r.pid assert r.output is None assert r.error is None assert "chatter hello &" in logged.pop()
def check_install_from_pypi(cli, delivery, package, simulate_version=None): cli.run("--debug", "-d%s" % delivery, "install", package) assert cli.succeeded assert cli.match("Installed %s" % package) assert runez.is_executable(package) m = TrackedManifest.from_file(dot_meta("%s/.manifest.json" % package)) assert str(m) assert m.entrypoints[package] assert m.install_info.args == runez.quoted(cli.args) assert m.install_info.timestamp assert m.install_info.vpickley == __version__ assert m.settings.delivery == delivery assert m.settings.python assert m.version r = runez.run(package, "--version") assert r.succeeded cli.expect_success("--debug auto-upgrade %s" % package, "Skipping auto-upgrade, checked recently") cli.expect_success("install %s" % package, "is already installed") cli.expect_success("check", "is installed") cli.expect_success("list", package) cli.expect_success("upgrade", "is already up-to-date") if simulate_version: m.version = simulate_version runez.save_json(m.to_dict(), dot_meta("%s/.manifest.json" % package)) cli.expect_success( "check", "v%s installed, can be upgraded to" % simulate_version)
def diff(compact, untyped, tokens, implementations, samples): """Compare deserialization of 2 implementations""" stringify = runez.stringified if untyped else decode if compact is None: compact = len(samples) > 1 with runez.TempFolder(): generated_files = [] for sample in samples: generated_files.append([sample]) for impl in implementations: assert isinstance(impl, Implementation) data = impl.get_outcome(sample, tokens=tokens) rep = TestSettings.represented(data, size=None, stringify=stringify, dt=simplified_date) fname = "%s-%s.txt" % (impl.name, sample.basename) generated_files[-1].extend([fname, rep]) if not compact: with open(fname, "w") as fh: fh.write(rep) if not rep.endswith("\n"): fh.write("\n") matches = 0 failed = 0 differ = 0 for sample, n1, r1, n2, r2 in generated_files: if isinstance(r1, dict) and isinstance( r2, dict) and r1.get("_error") and r2.get("_error"): matches += 1 failed += 1 print("%s: both failed" % sample) elif r1 == r2: matches += 1 print("%s: OK" % sample) else: differ += 1 if compact: print("%s: differ" % sample) if not compact: for sample, n1, r1, n2, r2 in generated_files: if r1 != r2: r = runez.run("diff", "-br", "-U1", n1, n2, fatal=None) print("======== %s ========" % sample) print(r.full_output) print() message = [ runez.plural(samples, "sample"), TestSettings.colored_if_meaningful(matches, "match", runez.green), TestSettings.colored_if_meaningful(differ, "differ", runez.orange), TestSettings.colored_if_meaningful(failed, "failed", runez.red), ] print("\n%s" % ", ".join(message))
def _run_builtin_module(self, mod, *args, **kwargs): args = runez.flattened(args, shellify=True) python = self.python if mod == "venv": # Use original python installation when using the builtin venv module python = self.venv_python.executable return runez.run(python, "-m%s" % mod, *args, **kwargs)
def sanity_check(self, args): """ :param str args: Args to run as sanity-check against all packaged exes, example: "--version" """ if args: for path in self.executables: result = runez.run(path, args) print("Sanity check: %s %s -> %s" % (short(path), args, result.full_output))
def __init__(self, lock, venv_python): """ :param SoftLock lock: Acquired lock """ self.venv_python = venv_python self.lock = lock self.folder = lock.folder self.bin = os.path.join(self.folder, "bin") self.python = os.path.join(self.bin, "python") self.pip = os.path.join(self.bin, "pip") self._frozen = None if runez.is_younger(self.python, self.lock.keep): return runez.delete(self.folder) venv = system.virtualenv_path() if not venv: runez.abort("Can't determine path to virtualenv.py") runez.run(self.venv_python.executable, venv, self.folder)
def is_healthily_installed(self): """Double-check that current venv is still usable""" py_path = self.cfg.available_pythons.resolved_python_exe( self.install_path) if py_path: return runez.run(py_path, "--version", dryrun=False, fatal=False, logger=False).succeeded
def test_package_venv(cli): # Verify that "debian mode" works as expected, with -droot/tmp <-> /tmp runez.delete("/tmp/pickley") cli.run("package", cli.project_folder, "-droot/tmp", "--no-compile", "--sanity-check=--version", "-sroot:root/usr/local/bin") assert cli.succeeded assert "--version" in cli.logged assert runez.is_executable("/tmp/pickley/bin/pickley") r = runez.run("/tmp/pickley/bin/pickley", "--version") assert r.succeeded runez.delete("/tmp/pickley")
def _set_executable(self, path): path = runez.resolved_path(path) if runez.is_executable(path): self.executable = path if not self.major or not self.minor: result = runez.run(self.executable, "--version", dryrun=False, fatal=False) if result.succeeded: m = RE_PYTHON_LOOSE.match(result.full_output) if m: self.major = m.group(3) self.minor = m.group(4)
def check_is_wrapper(path, is_wrapper): if is_wrapper: assert not os.path.islink(path) contents = runez.readlines(path) assert WRAPPER_MARK in contents else: assert os.path.islink(path) r = runez.run(path, "--version") assert r.succeeded
def __init__(self, lock, venv_python): """ :param SoftLock lock: Acquired lock """ self.venv_python = venv_python self.lock = lock self.folder = lock.folder self.bin = os.path.join(self.folder, "bin") self.python = os.path.join(self.bin, "python") self._frozen = None if runez.file.is_younger(self.python, self.lock.keep): return runez.delete(self.folder) is_py2 = runez.PY2 if venv_python: is_py2 = runez.to_int(venv_python.major, default=2) < 3 if is_py2: venv = virtualenv_path() if not venv: runez.abort("Can't determine path to virtualenv.py") runez.run(self.venv_python.executable, venv, self.folder) else: runez.run(self.venv_python.executable, "-mvenv", self.folder) runez.run(self.python, "-mpip", "install", "wheel")
def version_check(programs): """Check that programs are present with a minimum version""" if not programs: runez.abort("Specify at least one program to check") specs = [] for program_spec in programs: program, _, min_version = program_spec.partition(":") min_version = Version(min_version) if not program or not min_version.is_valid: runez.abort( "Invalid argument '%s', expecting format <program>:<version>" % program_spec) specs.append((program, min_version)) overview = [] for program, min_version in specs: if runez.DRYRUN: runez.run(program, "--version") continue full_path = runez.which(program) if not full_path: runez.abort("%s is not installed" % program) r = runez.run(full_path, "--version", fatal=False, logger=None) if not r.succeeded: runez.abort("%s --version failed: %s" % (runez.short(full_path), runez.short(r.full_output))) version = Version.from_text(r.full_output) if not version or version < min_version: runez.abort("%s version too low: %s (need %s+)" % (runez.short(full_path), version, min_version)) overview.append("%s %s" % (program, version)) print(runez.short(runez.joined(overview, delimiter=" ; ")))
def validate_sanity_check(exe, sanity_check): if not exe or not sanity_check: return None r = runez.run(exe, sanity_check, fatal=False) if r.failed: if does_not_implement_cli_flag(r.output, r.error): return "does not respond to %s" % sanity_check abort("'%s' failed %s sanity check: %s" % (exe, sanity_check, r.full_output)) return runez.first_line(r.output or r.error)
def __init__(self, pspec=None, folder=None, python=None, index=None, cfg=None, create=True): """ Args: pspec (pickley.PackageSpec | None): Package spec to install folder (str | None): Target folder (default: pspec.install_path) python (pickley.env.PythonInstallation): Python to use (default: pspec.python) index (str | None): Optional custom pypi index to use (default: pspec.index) cfg (pickley.PickleyConfig | None): Config to use create (bool): Create venv if True """ if folder is None and pspec: folder = pspec.install_path if not python and pspec: python = pspec.python if not index and pspec: index = pspec.index self.folder = folder self.index = index self.py_path = os.path.join(folder, "bin", "python3") self.pip_path = os.path.join(folder, "bin", "pip3") if create and folder: cfg = cfg or pspec.cfg python = python or cfg.find_python(pspec=pspec) runez.ensure_folder(folder, clean=True, logger=False) if python.version > "3.7": runez.run(python.executable, "-mvenv", folder) else: import virtualenv.__main__ runez.run(sys.executable, virtualenv.__main__.__file__, "-p", python.executable, folder) self.run_pip("install", "-U", "pip")
def _run_from_venv(self, command, *args, **kwargs): """ Should be called while holding the soft file lock in context only :param str command: Command to run from that package (optionally specced with version) :param args: Args to invoke program with :param kwargs: Additional args """ cmd = system.PackageSpec(command) if cmd.dashed in ("pip", "venv"): return self._run_builtin_module(cmd.dashed, *args, **kwargs) args = runez.flattened(args, shellify=True) full_path = self._installed_module(cmd) return runez.run(full_path, *args, **kwargs)
def _run_from_venv(self, package_name, *args, **kwargs): """ Should be called while holding the soft file lock in context only :param str package_name: Pypi package to which command being ran belongs to :param args: Args to invoke program with :param kwargs: Additional args, use program= if entry point differs from 'package_name' """ if package_name == "pip": return self._run_pip(*args, **kwargs) args = runez.flattened(args, split=runez.SHELL) program = kwargs.pop("program", package_name) program, version = system.despecced(program) full_path = self._installed_module(program, version=version) return runez.run(full_path, *args, **kwargs)
def _set_executable(self, path): path = runez.resolved_path(path) if runez.is_executable(path): self.executable = path if not self.major or not self.minor: output = runez.run(self.executable, "--version", dryrun=False, fatal=None, include_error=True, logger=None) if output: m = RE_PYTHON_LOOSE.match(output) if m: self.major = m.group(3) self.minor = m.group(4)
def install(self): self.validate(require_installed=False) if os.path.isfile(self.target_docker_compose): print("%s is already installed" % self.name) return if not os.path.isdir(self.target): runez.run("sudo", "mkdir", self.target) runez.run("sudo", "chown", os.environ.get("USER"), self.target) runez.run("rsync", "-aHJ", "%s/" % self.origin, self.target)
def cmd_passthrough(): """ Capture pass-through test Run a program, capture its output as well as let it pass-through to stdout/stderr """ parser = runez.cli.parser() args, unknown = parser.parse_known_args() unknown = runez.flattened(unknown, split=" ") if not unknown: sys.exit("Provide command to run") print("-- Running: %s\n" % unknown) r = runez.run(*unknown, fatal=False, passthrough=True) print("\n---- Captured: (exit code %s) ----" % r.exit_code) print("\nstdout:\n%s" % (r.output or runez.dim("-empty-"))) print("\nstderr:\n%s" % (r.error or runez.dim("-empty-")))
def brew_uninstall(target, fatal=False): """ :param str target: Path of file to uninstall :param bool fatal: Abort if True :return int: 1 if successfully uninstalled, 0 if nothing to do, -1 if failed """ brew, name = find_brew_name(target) if not brew or not name: return -1 result = runez.run(brew, "uninstall", "-f", name, fatal=False, logger=LOG.info) if result.failed: # Failed brew uninstall return runez.abort("'%s uninstall %s' failed, please check", brew, name, fatal=(fatal, -1)) # All good return 1