def resolved_name_version(self):
if not self.folder:
if self.version:
return self.original, self.version
return despecced(self.original)
setup_py = os.path.join(self.folder, "setup.py")
if not os.path.exists(setup_py):
abort("No setup.py in %s" % self.folder)
with runez.CurrentFolder(self.folder):
# Some setup.py's assume current folder is the one with their setup.py
r = runez.run(sys.executable,
"setup.py",
"--name",
dryrun=False,
fatal=False,
logger=False)
package_name = r.output
if r.failed or not package_name:
abort("Could not determine package name from %s" % setup_py)
validate_pypi_name(package_name)
r = runez.run(sys.executable,
"setup.py",
"--version",
dryrun=False,
fatal=False,
logger=False)
package_version = r.output
if r.failed or not package_version:
abort("Could not determine package version from setup.py")
return package_name, package_version
def effective_package(self, template):
"""
:param str template: Template describing how to name delivered files, example: {meta}/{name}-{version}
"""
folder = os.path.join(self.dist_folder, template.format(name=self.package_spec.dashed, version=self.desired.version))
clean_folder(folder)
python = system.target_python(package_spec=self.package_spec)
if not python.has_builtin_venv or self.relocatable:
venv = "virtualenv==16.7.7"
else:
venv = "venv"
vrun(self.package_spec, venv, folder)
bin_folder = os.path.join(folder, "bin")
pip = os.path.join(bin_folder, "pip")
spec = self.source_folder if self.source_folder else "%s==%s" % (self.package_spec.dashed, self.desired.version)
runez.run(pip, "install", "-i", system.SETTINGS.index, "-f", self.build_folder, spec)
if self.relocatable:
python = system.target_python(package_spec=self.package_spec).executable
vrun(self.package_spec, "virtualenv", "--relocatable", "--python=%s" % python, folder)
runez.run(os.path.join(bin_folder, "python"), "-mcompileall")
self.packaged.append(folder)
self.executables = [os.path.join(bin_folder, name) for name in self.entry_points]
def test_run(temp_folder):
assert runez.program.added_env_paths(None) is None
ls = runez.which("ls")
with runez.CaptureOutput(dryrun=True) as logged:
assert "Would run: /dev/null" in runez.run("/dev/null", fatal=False)
assert "Would run: /dev/null" in logged.pop()
assert "Would run:" in runez.run(ls, "--invalid-flag", None, ".")
assert "Would run: %s ." % ls in logged.pop()
with runez.CaptureOutput() as logged:
assert runez.run("/dev/null", fatal=False) is False
assert "/dev/null is not installed" in logged.pop()
assert runez.touch("sample") == 1
assert runez.run(ls,
"--invalid-flag",
None,
".",
path_env={"PATH": ":."}) == "sample"
assert "Running: %s ." % ls in logged.pop()
assert runez.run(ls, "some-file", fatal=False) is False
assert "Running: %s some-file" % ls in logged
assert "exited with code" in logged
assert "No such file" in logged.pop()
def effective_package(self, template):
"""
:param str template: Template describing how to name delivered files, example: {meta}/{name}-{version}
"""
folder = os.path.join(
self.dist_folder,
template.format(name=self.name, version=self.version))
runez.delete(folder, logger=None)
runez.ensure_folder(folder, folder=True, logger=None)
vrun(self.name, "virtualenv", folder)
bin_folder = os.path.join(folder, "bin")
pip = os.path.join(bin_folder, "pip")
spec = self.source_folder if self.source_folder else "%s==%s" % (
self.name, self.version)
runez.run(pip, "install", "-i", system.SETTINGS.index, "-f",
self.build_folder, spec)
if self.relocatable:
python = system.target_python(package_name=self.name).executable
vrun(self.name, "virtualenv", "--relocatable",
"--python=%s" % python, folder)
self.packaged.append(folder)
self.executables = [
os.path.join(bin_folder, name) for name in self.entry_points
]
def upgrade(self):
self.stop()
with runez.CurrentFolder(self.target):
runez.run("docker-compose", "pull")
runez.run("docker-compose", "prune", "-f")
self.sync()
self.start()
def git_clone(pspec):
basename = runez.basename(pspec.original)
pspec.folder = pspec.cfg.cache.full_path("checkout", basename)
runez.ensure_folder(pspec.folder, clean=True, dryrun=False)
logger = LOG.info if runez.DRYRUN else runez.UNSET
runez.run("git",
"clone",
pspec.original,
pspec.folder,
dryrun=False,
logger=logger)
def test_capture(temp_folder, logged):
chatter = runez.resolved_path("chatter")
assert runez.write(chatter, CHATTER.strip(), fatal=False) == 1
assert runez.make_executable(chatter, fatal=False) == 1
assert runez.run(chatter, fatal=False) == "chatter"
r = runez.run(chatter, include_error=True, fatal=False)
assert r.startswith("chatter")
assert "No such file" in r
assert "Running: chatter" in logged
def stop_profiler(cls):
cls.profiler.disable()
filepath = runez.DEV.project_path(".tox", "lastrun.profile")
try:
cls.profiler.dump_stats(filepath)
if runez.which("qcachegrind") is None:
print("run 'brew install qcachegrind'")
return
runez.run("pyprof2calltree", "-k", "-i", filepath, stdout=None, stderr=None)
except Exception as e:
print("Can't save %s: %s" % (filepath, e))
def check_wrap(self, wrap_method):
impl = DeliveryMethod.delivery_method_by_name(wrap_method)
impl.install(self.pspec, self.venv, self.entry_points)
exe = runez.resolved_path(self.name)
r = runez.run(exe, "--version", fatal=False)
assert r.succeeded
assert r.output == self.version
def request_get(url):
"""
:param str url: URL to query
:return str: Response body
"""
try:
LOG.debug("GET %s", url)
request = Request(url) # nosec
response = urlopen(request).read() # nosec
return response and runez.decode(response).strip()
except Exception as e:
code = getattr(e, "code", None)
if isinstance(code, int) and 400 <= code < 500:
return None
try:
# Some old python installations have trouble with SSL (OSX for example), try curl
result = runez.run("curl", "-s", url, dryrun=False, fatal=False)
if result.succeeded and result.output:
return result.output
except Exception as e:
LOG.debug("GET %s failed: %s", url, e, exc_info=e)
return None
def test_background_run(logged):
with runez.CurrentFolder(os.path.dirname(CHATTER)):
r = runez.run(CHATTER,
"hello",
background=True,
dryrun=True,
logger=True)
assert r.succeeded
assert "chatter hello &" in logged.pop()
r = runez.run(CHATTER, "hello", background=True, dryrun=False)
assert r.succeeded
assert r.pid
assert r.output is None
assert r.error is None
assert "chatter hello &" in logged.pop()
def check_install_from_pypi(cli, delivery, package, simulate_version=None):
cli.run("--debug", "-d%s" % delivery, "install", package)
assert cli.succeeded
assert cli.match("Installed %s" % package)
assert runez.is_executable(package)
m = TrackedManifest.from_file(dot_meta("%s/.manifest.json" % package))
assert str(m)
assert m.entrypoints[package]
assert m.install_info.args == runez.quoted(cli.args)
assert m.install_info.timestamp
assert m.install_info.vpickley == __version__
assert m.settings.delivery == delivery
assert m.settings.python
assert m.version
r = runez.run(package, "--version")
assert r.succeeded
cli.expect_success("--debug auto-upgrade %s" % package,
"Skipping auto-upgrade, checked recently")
cli.expect_success("install %s" % package, "is already installed")
cli.expect_success("check", "is installed")
cli.expect_success("list", package)
cli.expect_success("upgrade", "is already up-to-date")
if simulate_version:
m.version = simulate_version
runez.save_json(m.to_dict(), dot_meta("%s/.manifest.json" % package))
cli.expect_success(
"check", "v%s installed, can be upgraded to" % simulate_version)
def diff(compact, untyped, tokens, implementations, samples):
"""Compare deserialization of 2 implementations"""
stringify = runez.stringified if untyped else decode
if compact is None:
compact = len(samples) > 1
with runez.TempFolder():
generated_files = []
for sample in samples:
generated_files.append([sample])
for impl in implementations:
assert isinstance(impl, Implementation)
data = impl.get_outcome(sample, tokens=tokens)
rep = TestSettings.represented(data,
size=None,
stringify=stringify,
dt=simplified_date)
fname = "%s-%s.txt" % (impl.name, sample.basename)
generated_files[-1].extend([fname, rep])
if not compact:
with open(fname, "w") as fh:
fh.write(rep)
if not rep.endswith("\n"):
fh.write("\n")
matches = 0
failed = 0
differ = 0
for sample, n1, r1, n2, r2 in generated_files:
if isinstance(r1, dict) and isinstance(
r2, dict) and r1.get("_error") and r2.get("_error"):
matches += 1
failed += 1
print("%s: both failed" % sample)
elif r1 == r2:
matches += 1
print("%s: OK" % sample)
else:
differ += 1
if compact:
print("%s: differ" % sample)
if not compact:
for sample, n1, r1, n2, r2 in generated_files:
if r1 != r2:
r = runez.run("diff", "-br", "-U1", n1, n2, fatal=None)
print("======== %s ========" % sample)
print(r.full_output)
print()
message = [
runez.plural(samples, "sample"),
TestSettings.colored_if_meaningful(matches, "match", runez.green),
TestSettings.colored_if_meaningful(differ, "differ", runez.orange),
TestSettings.colored_if_meaningful(failed, "failed", runez.red),
]
print("\n%s" % ", ".join(message))
def _run_builtin_module(self, mod, *args, **kwargs):
args = runez.flattened(args, shellify=True)
python = self.python
if mod == "venv":
# Use original python installation when using the builtin venv module
python = self.venv_python.executable
return runez.run(python, "-m%s" % mod, *args, **kwargs)
def sanity_check(self, args):
"""
:param str args: Args to run as sanity-check against all packaged exes, example: "--version"
"""
if args:
for path in self.executables:
result = runez.run(path, args)
print("Sanity check: %s %s -> %s" % (short(path), args, result.full_output))
def __init__(self, lock, venv_python):
"""
:param SoftLock lock: Acquired lock
"""
self.venv_python = venv_python
self.lock = lock
self.folder = lock.folder
self.bin = os.path.join(self.folder, "bin")
self.python = os.path.join(self.bin, "python")
self.pip = os.path.join(self.bin, "pip")
self._frozen = None
if runez.is_younger(self.python, self.lock.keep):
return
runez.delete(self.folder)
venv = system.virtualenv_path()
if not venv:
runez.abort("Can't determine path to virtualenv.py")
runez.run(self.venv_python.executable, venv, self.folder)
def is_healthily_installed(self):
"""Double-check that current venv is still usable"""
py_path = self.cfg.available_pythons.resolved_python_exe(
self.install_path)
if py_path:
return runez.run(py_path,
"--version",
dryrun=False,
fatal=False,
logger=False).succeeded
def test_package_venv(cli):
# Verify that "debian mode" works as expected, with -droot/tmp <-> /tmp
runez.delete("/tmp/pickley")
cli.run("package", cli.project_folder, "-droot/tmp", "--no-compile",
"--sanity-check=--version", "-sroot:root/usr/local/bin")
assert cli.succeeded
assert "--version" in cli.logged
assert runez.is_executable("/tmp/pickley/bin/pickley")
r = runez.run("/tmp/pickley/bin/pickley", "--version")
assert r.succeeded
runez.delete("/tmp/pickley")
def _set_executable(self, path):
path = runez.resolved_path(path)
if runez.is_executable(path):
self.executable = path
if not self.major or not self.minor:
result = runez.run(self.executable, "--version", dryrun=False, fatal=False)
if result.succeeded:
m = RE_PYTHON_LOOSE.match(result.full_output)
if m:
self.major = m.group(3)
self.minor = m.group(4)
def check_is_wrapper(path, is_wrapper):
if is_wrapper:
assert not os.path.islink(path)
contents = runez.readlines(path)
assert WRAPPER_MARK in contents
else:
assert os.path.islink(path)
r = runez.run(path, "--version")
assert r.succeeded
def __init__(self, lock, venv_python):
"""
:param SoftLock lock: Acquired lock
"""
self.venv_python = venv_python
self.lock = lock
self.folder = lock.folder
self.bin = os.path.join(self.folder, "bin")
self.python = os.path.join(self.bin, "python")
self._frozen = None
if runez.file.is_younger(self.python, self.lock.keep):
return
runez.delete(self.folder)
is_py2 = runez.PY2
if venv_python:
is_py2 = runez.to_int(venv_python.major, default=2) < 3
if is_py2:
venv = virtualenv_path()
if not venv:
runez.abort("Can't determine path to virtualenv.py")
runez.run(self.venv_python.executable, venv, self.folder)
else:
runez.run(self.venv_python.executable, "-mvenv", self.folder)
runez.run(self.python, "-mpip", "install", "wheel")
def version_check(programs):
"""Check that programs are present with a minimum version"""
if not programs:
runez.abort("Specify at least one program to check")
specs = []
for program_spec in programs:
program, _, min_version = program_spec.partition(":")
min_version = Version(min_version)
if not program or not min_version.is_valid:
runez.abort(
"Invalid argument '%s', expecting format <program>:<version>" %
program_spec)
specs.append((program, min_version))
overview = []
for program, min_version in specs:
if runez.DRYRUN:
runez.run(program, "--version")
continue
full_path = runez.which(program)
if not full_path:
runez.abort("%s is not installed" % program)
r = runez.run(full_path, "--version", fatal=False, logger=None)
if not r.succeeded:
runez.abort("%s --version failed: %s" %
(runez.short(full_path), runez.short(r.full_output)))
version = Version.from_text(r.full_output)
if not version or version < min_version:
runez.abort("%s version too low: %s (need %s+)" %
(runez.short(full_path), version, min_version))
overview.append("%s %s" % (program, version))
print(runez.short(runez.joined(overview, delimiter=" ; ")))
def validate_sanity_check(exe, sanity_check):
if not exe or not sanity_check:
return None
r = runez.run(exe, sanity_check, fatal=False)
if r.failed:
if does_not_implement_cli_flag(r.output, r.error):
return "does not respond to %s" % sanity_check
abort("'%s' failed %s sanity check: %s" %
(exe, sanity_check, r.full_output))
return runez.first_line(r.output or r.error)
def __init__(self, pspec=None, folder=None, python=None, index=None, cfg=None, create=True):
"""
Args:
pspec (pickley.PackageSpec | None): Package spec to install
folder (str | None): Target folder (default: pspec.install_path)
python (pickley.env.PythonInstallation): Python to use (default: pspec.python)
index (str | None): Optional custom pypi index to use (default: pspec.index)
cfg (pickley.PickleyConfig | None): Config to use
create (bool): Create venv if True
"""
if folder is None and pspec:
folder = pspec.install_path
if not python and pspec:
python = pspec.python
if not index and pspec:
index = pspec.index
self.folder = folder
self.index = index
self.py_path = os.path.join(folder, "bin", "python3")
self.pip_path = os.path.join(folder, "bin", "pip3")
if create and folder:
cfg = cfg or pspec.cfg
python = python or cfg.find_python(pspec=pspec)
runez.ensure_folder(folder, clean=True, logger=False)
if python.version > "3.7":
runez.run(python.executable, "-mvenv", folder)
else:
import virtualenv.__main__
runez.run(sys.executable, virtualenv.__main__.__file__, "-p", python.executable, folder)
self.run_pip("install", "-U", "pip")
def _run_from_venv(self, command, *args, **kwargs):
"""
Should be called while holding the soft file lock in context only
:param str command: Command to run from that package (optionally specced with version)
:param args: Args to invoke program with
:param kwargs: Additional args
"""
cmd = system.PackageSpec(command)
if cmd.dashed in ("pip", "venv"):
return self._run_builtin_module(cmd.dashed, *args, **kwargs)
args = runez.flattened(args, shellify=True)
full_path = self._installed_module(cmd)
return runez.run(full_path, *args, **kwargs)
def _run_from_venv(self, package_name, *args, **kwargs):
"""
Should be called while holding the soft file lock in context only
:param str package_name: Pypi package to which command being ran belongs to
:param args: Args to invoke program with
:param kwargs: Additional args, use program= if entry point differs from 'package_name'
"""
if package_name == "pip":
return self._run_pip(*args, **kwargs)
args = runez.flattened(args, split=runez.SHELL)
program = kwargs.pop("program", package_name)
program, version = system.despecced(program)
full_path = self._installed_module(program, version=version)
return runez.run(full_path, *args, **kwargs)
def _set_executable(self, path):
path = runez.resolved_path(path)
if runez.is_executable(path):
self.executable = path
if not self.major or not self.minor:
output = runez.run(self.executable,
"--version",
dryrun=False,
fatal=None,
include_error=True,
logger=None)
if output:
m = RE_PYTHON_LOOSE.match(output)
if m:
self.major = m.group(3)
self.minor = m.group(4)
def install(self):
self.validate(require_installed=False)
if os.path.isfile(self.target_docker_compose):
print("%s is already installed" % self.name)
return
if not os.path.isdir(self.target):
runez.run("sudo", "mkdir", self.target)
runez.run("sudo", "chown", os.environ.get("USER"), self.target)
runez.run("rsync", "-aHJ", "%s/" % self.origin, self.target)
def cmd_passthrough():
"""
Capture pass-through test
Run a program, capture its output as well as let it pass-through to stdout/stderr
"""
parser = runez.cli.parser()
args, unknown = parser.parse_known_args()
unknown = runez.flattened(unknown, split=" ")
if not unknown:
sys.exit("Provide command to run")
print("-- Running: %s\n" % unknown)
r = runez.run(*unknown, fatal=False, passthrough=True)
print("\n---- Captured: (exit code %s) ----" % r.exit_code)
print("\nstdout:\n%s" % (r.output or runez.dim("-empty-")))
print("\nstderr:\n%s" % (r.error or runez.dim("-empty-")))
def brew_uninstall(target, fatal=False):
"""
:param str target: Path of file to uninstall
:param bool fatal: Abort if True
:return int: 1 if successfully uninstalled, 0 if nothing to do, -1 if failed
"""
brew, name = find_brew_name(target)
if not brew or not name:
return -1
result = runez.run(brew, "uninstall", "-f", name, fatal=False, logger=LOG.info)
if result.failed:
# Failed brew uninstall
return runez.abort("'%s uninstall %s' failed, please check", brew, name, fatal=(fatal, -1))
# All good
return 1
