def resolve(self):
if not os.path.isdir(self.folder):
abort("Folder %s does not exist" %
runez.red(runez.short(self.folder)))
req = self.requirements
if not req:
default_req = runez.resolved_path("requirements.txt",
base=self.folder)
if os.path.exists(default_req):
req = [default_req]
if req:
req = [("-r", runez.resolved_path(r, base=self.folder))
for r in req]
req = runez.flattened(req, shellify=True)
req.append(self.folder)
self.requirements = req
self.pspec = PackageSpec(CFG, self.folder)
LOG.info("Using python: %s" % self.pspec.python)
if self.dist.startswith("root/"):
# Special case: we're targeting 'root/...' probably for a debian, use target in that case to avoid venv relocation issues
target = self.dist[4:]
if os.path.isdir(target):
LOG.debug("debian mode: %s -> %s", self.dist, target)
self.dist = target
parts = self.dist.split("/")
if len(parts) <= 2:
# Auto-add package name to targets of the form root/subfolder (most typical case)
self.dist = os.path.join(self.dist, self.pspec.dashed)
def mocked_invoker(**sysattrs):
major = sysattrs.pop("major", 3)
pyenv = sysattrs.pop("pyenv", None)
use_path = sysattrs.pop("use_path", False)
exe_exists = sysattrs.pop("exe_exists", True)
sysattrs.setdefault("base_prefix", "/usr")
sysattrs.setdefault("prefix", sysattrs["base_prefix"])
sysattrs["base_prefix"] = runez.resolved_path(sysattrs["base_prefix"])
sysattrs["prefix"] = runez.resolved_path(sysattrs["prefix"])
sysattrs.setdefault("executable",
"%s/bin/python" % sysattrs["base_prefix"])
sysattrs.setdefault("version_info", (major, 7, 1))
sysattrs.setdefault("version",
".".join(str(s) for s in sysattrs["version_info"]))
scanner = None if not pyenv else PythonInstallationScanner(pyenv)
with patch("runez.pyenv.os.path.realpath", side_effect=lambda x: x):
with patch("runez.pyenv.sys") as mocked:
for k, v in sysattrs.items():
setattr(mocked, k, v)
if isinstance(exe_exists, bool):
with patch("runez.pyenv.is_executable",
return_value=exe_exists):
return PythonDepot(scanner=scanner, use_path=use_path)
with patch("runez.pyenv.is_executable", side_effect=exe_exists):
return PythonDepot(scanner=scanner, use_path=use_path)
def __init__(self, spec):
self.base, _, self.target = spec.partition(":")
if not self.base or not self.target:
abort("Invalid symlink specification '%s'" % spec)
self.base = runez.resolved_path(self.base)
self.target = runez.resolved_path(self.target)
def create_symlinks(self, symlink, root=None, fatal=True):
"""
Use case: preparing a .tox/package/root folder to be packaged as a debian
With a spec of "root:root/usr/local/bin", all executables produced under ./root will be symlinked to /usr/local/bin
:param str symlink: A specification of the form "root:root/usr/local/bin"
:param str root: Optionally, 'root' prefix if used
:param bool fatal: Abort execution on failure if True
:return int: 1 if effectively done, 0 if no-op, -1 on failure
"""
if not symlink or not self.executables:
return 0
base, _, target = symlink.partition(":")
if not target:
return runez.abort("Invalid symlink specification '%s'", symlink, fatal=(fatal, -1))
base = runez.resolved_path(base)
target = runez.resolved_path(target)
for path in self.executables:
if path and root:
path = runez.resolved_path(root + path)
if not path.startswith(base) or len(path) <= len(base):
return runez.abort("Symlink base '%s' does not cover '%s'", base, path, fatal=(fatal, -1))
source = path[len(base):]
basename = os.path.basename(path)
destination = os.path.join(target, basename)
runez.symlink(source, destination, must_exist=False, fatal=fatal, logger=LOG.info)
return 1 if self.executables else 0
def package(build, dist, symlink, relocatable, sanity_check, folder):
"""
Package a project from source checkout
"""
build = runez.resolved_path(build)
root = None
target_dist = dist
if target_dist.startswith("root/"):
# Special case: we're targeting 'root/...' probably for a debian, use target in that case to avoid venv relocation issues
target = target_dist[4:]
if os.path.isdir(target):
root = target_dist[:4]
target_dist = target
LOG.debug("debian mode: %s -> %s", dist, target)
folder = runez.resolved_path(folder)
if not os.path.isdir(folder):
sys.exit("Folder %s does not exist" % short(folder))
system.SETTINGS.set_base(build)
setup_py = os.path.join(folder, "setup.py")
if not os.path.exists(setup_py):
sys.exit("No setup.py in %s" % short(folder))
with runez.CurrentFolder(folder):
# Some setup.py's assume their working folder is the folder where they're in
result = system.run_python(setup_py,
"--name",
fatal=False,
dryrun=False)
name = result.output
if result.failed or not name:
sys.exit("Could not determine package name from %s" %
short(setup_py))
package_spec = system.PackageSpec(name)
runez.Anchored.add(folder)
p = PACKAGERS.resolved(package_spec)
p.build_folder = build
p.dist_folder = runez.resolved_path(target_dist)
p.relocatable = relocatable
p.source_folder = folder
p.package()
p.create_symlinks(symlink, root=root)
p.sanity_check(sanity_check)
if p.executables:
overview = "produced: %s" % runez.quoted(p.executables)
else:
overview = "package has no entry-points"
print("Packaged %s successfully, %s" % (short(folder), overview))
runez.Anchored.pop(folder)
def test_anchored():
user_path = runez.resolved_path("~/some-folder/bar")
current_path = runez.resolved_path("./some-folder/bar")
assert user_path != "~/some-folder/bar"
assert runez.short(user_path) == "~/some-folder/bar"
assert runez.short(current_path) != "some-folder/bar"
with runez.Anchored(os.getcwd()):
assert runez.short(current_path) == "some-folder/bar"
def test_path_resolution(temp_folder):
assert runez.resolved_path(None) is None
assert runez.resolved_path("some-file") == os.path.join(
temp_folder, "some-file")
assert runez.resolved_path("some-file", base="bar") == os.path.join(
temp_folder, "bar", "some-file")
assert runez.quoted(
["ls",
os.path.join(temp_folder, "some-file") + " bar", "-a",
" foo "]) == 'ls "some-file bar" -a " foo "'
def test_shortening():
assert runez.short(None) == "None"
assert runez.short("") == ""
assert runez.short(5) == "5"
assert runez.short(" some text ") == "some text"
assert runez.short(" \n some \n long text", size=9) == "some l..."
assert runez.short(" \n some \n long text", size=8) == "some ..."
assert runez.short(" a \n\n \n b ") == "a b"
assert runez.short([1, "b"]) == "[1, b]"
assert runez.short((1, {
"b": ["c", {"d", "e"}]
})) == "(1, {b: [c, {d, e}]})"
complex = {
"a \n b":
[1, None, "foo \n ,", {
"a2": runez.abort,
"c": runez.Anchored
}],
None: datetime.date(2019, 1, 1)
}
assert runez.short(
complex
) == "{None: 2019-01-01, a b: [1, None, foo ,, {a2: function 'abort', c: class runez.system.Anchored}]}"
assert runez.short(complex, size=32) == "{None: 2019-01-01, a b: [1, N..."
assert runez.short(" some text ", size=32) == "some text"
assert runez.short(" some text ", size=7) == "some..."
assert runez.short(" some text ", size=0) == "some text"
# Verify that coloring is not randomly truncated
assert runez.short("\033[38;2;255;0;0mfoo bar baz\033[39m",
size=6,
uncolor=True) == "foo..."
with runez.TempFolder() as tmp:
assert runez.short(os.path.join(tmp, "some-file")) == "some-file"
user_path = runez.resolved_path("~/some-folder/bar")
current_path = runez.resolved_path("./some-folder/bar")
assert user_path != "~/some-folder/bar"
assert runez.short(user_path) == "~/some-folder/bar"
assert runez.short(current_path) == "some-folder/bar"
with runez.Anchored(os.getcwd(), "./foo"):
assert runez.short(current_path) == os.path.join(
"some-folder", "bar")
assert runez.short("./foo") == "./foo"
assert runez.short(runez.resolved_path("foo")) == "foo"
assert runez.short(runez.resolved_path("./foo/bar")) == "bar"
assert not runez.Anchored._paths
def find_base():
base_path = runez.resolved_path(os.environ.get("PICKLEY_ROOT"))
if base_path:
if not os.path.isdir(base_path):
abort("PICKLEY_ROOT points to non-existing directory %s" %
runez.red(base_path))
return runez.resolved_path(base_path)
program_path = PickleyConfig.program_path
return _find_base_from_program_path(program_path) or os.path.dirname(
program_path)
def _resolve_from_configured(self, folder):
"""
Resolve python executable from a configured folder
This aims to support pyenv like installations, as well as /usr/bin-like ones
"""
folder = runez.resolved_path(folder)
if not folder or not self.major or not os.path.isdir(folder):
return None
timestamp = None
result = None
interesting = [
self.program_name,
"%s.%s" % (self.major, self.minor or "")
]
for fname in os.listdir(folder):
m = RE_PYTHON_STRICT.match(fname)
if not m:
continue
m = m.group(2) or m.group(3)
if not m or not any(m.startswith(x) for x in interesting):
continue
ts = os.path.getmtime(os.path.join(folder, fname))
if timestamp is None or timestamp < ts:
timestamp = ts
result = fname
return result and self._first_executable(
os.path.join(folder, result, "bin", "python"),
os.path.join(folder, result),
)
def mk_python(basename,
prefix=None,
base_prefix=None,
executable=True,
content=None,
folder=None,
version=None):
if version is None:
m = RE_VERSION.match(basename)
if m:
if not folder:
folder = os.path.join(".pyenv/versions", basename)
version = m.group(2)
basename = "python"
if not folder:
folder = ".pyenv/versions"
path = runez.resolved_path(folder)
if not prefix:
prefix = path
if not base_prefix:
base_prefix = prefix
path = os.path.join(path, "bin", basename)
if not content:
content = [version, prefix, base_prefix]
content = "#!/bin/bash\n%s\n" % "\n".join("echo %s" % s for s in content)
runez.write(path, content, logger=None)
if executable:
runez.make_executable(path, logger=None)
def test_pathlib(temp_folder):
subfolder = Path("subfolder")
assert runez.to_path(subfolder) is subfolder
assert not subfolder.is_dir()
runez.ensure_folder(subfolder)
assert subfolder.is_dir()
with pytest.raises(Exception):
runez.to_path("foo bar", no_spaces=Exception)
with runez.CurrentFolder(subfolder, anchor=True):
path = Path("foo")
assert runez.short(path) == "foo"
assert runez.short(path.absolute()) == "foo"
assert runez.resolved_path(path)
assert runez.parent_folder(path) == os.path.join(temp_folder, "subfolder")
assert runez.touch(path) == 1
assert runez.copy(path, Path("bar")) == 1
assert runez.copy(Path("bar"), Path("baz")) == 1
foo_json = Path("foo.json")
runez.write(path, '{"a": "b"}')
runez.symlink(path, foo_json)
assert runez.read_json(foo_json) == {"a": "b"}
assert list(runez.readlines(foo_json)) == ['{"a": "b"}']
assert runez.basename(foo_json.absolute()) == "foo"
def main(ctx, debug, base, index, config, python, delivery, packager):
"""
Package manager for python CLIs
"""
if any(opt in sys.argv
for opt in ctx.help_option_names): # pragma: no cover
return
runez.log.setup(
debug=debug,
console_format="%(levelname)s %(message)s" if debug else "%(message)s",
console_level=logging.WARNING,
console_stream=sys.stderr,
locations=None,
)
# Disable logging.config, as pip tries to get smart and configure all logging...
runez.log.silence("pip")
logging.config.dictConfig = lambda x: None
if base:
base = runez.resolved_path(base)
if not os.path.exists(base):
sys.exit("Can't use %s as base: folder does not exist" %
short(base))
system.SETTINGS.set_base(base)
system.SETTINGS.load_config(config=config,
delivery=delivery,
index=index,
packager=packager)
system.DESIRED_PYTHON = python
def finalize(self):
"""Run sanity check and/or symlinks, and return a report"""
with runez.Anchored(self.folder, CFG.base.path):
runez.ensure_folder(CFG.base.path, clean=True, logger=False)
dist_folder = runez.resolved_path(self.dist)
exes = PACKAGER.package(self.pspec, CFG.base.path, dist_folder,
self.requirements, self.compile)
if exes:
report = PrettyTable(
["Packaged executable", self.sanity_check],
border=self.border)
report.header.style = "bold"
if not self.sanity_check:
report.header[1].shown = False
for exe in exes:
exe_info = self.validate_sanity_check(
exe, self.sanity_check)
report.add_row(runez.quoted(exe), exe_info)
if self.symlink and exe:
self.symlink.apply(exe)
if not self.compile and not runez.DRYRUN:
clean_compiled_artifacts(dist_folder)
return report
def __init__(self, path, name=None):
"""
:param str path: Path to folder
:param str|None name: Name of this folder (defaults to basename of 'path')
"""
self.path = runez.resolved_path(path)
self.name = name or os.path.basename(path)
def check_wrap(self, wrap_method):
impl = DeliveryMethod.delivery_method_by_name(wrap_method)
impl.install(self.pspec, self.venv, self.entry_points)
exe = runez.resolved_path(self.name)
r = runez.run(exe, "--version", fatal=False)
assert r.succeeded
assert r.output == self.version
def test_base(cli, monkeypatch):
monkeypatch.setenv("__PYVENV_LAUNCHER__", "foo")
folder = os.getcwd()
cli.expect_success("-n base", folder)
cli.expect_success("-n base audit", dot_meta("audit.log", parent=folder))
cli.expect_success("-n base cache", dot_meta(".cache", parent=folder))
cli.expect_success("-n base meta", dot_meta(parent=folder))
cli.expect_failure("-n base foo", "Unknown base folder reference")
cli.run("-n base bootstrap-own-wrapper")
assert cli.succeeded
assert "Would wrap pickley" in cli.logged
monkeypatch.setenv("PICKLEY_ROOT", "temp-base")
with pytest.raises(SystemExit): # Env var points to a non-existing folder
find_base()
runez.ensure_folder("temp-base")
assert find_base() == runez.resolved_path("temp-base")
assert sys.prefix in get_program_path("foo/bar.py")
monkeypatch.delenv("PICKLEY_ROOT")
monkeypatch.setattr(PickleyConfig, "program_path",
"/foo/.venv/bin/pickley")
assert find_base() == "/foo/.venv/root"
monkeypatch.setattr(PickleyConfig, "program_path",
dot_meta("pickley-0.0.0/bin/pickley", parent="foo"))
assert find_base() == "foo"
monkeypatch.setattr(PickleyConfig, "program_path", "foo/bar")
assert find_base() == "foo"
def __init__(self, name, path):
"""
Args:
name (str): Internal name of this folder
path (str): Path to folder
"""
self.name = name
self.path = runez.resolved_path(path)
def _get_delayed_resolver(self):
if "://" in self.original or self.original.endswith(".git"):
return git_clone
if "/" in self.original:
folder = runez.resolved_path(self.original)
if os.path.isdir(folder):
self.folder = folder
def __init__(self, default):
path = runez.DEV.venv_path(os.path.basename(default))
if not path:
path = os.environ.get("GDOT_GIT_STORE")
if not path:
path = default
self.path = runez.resolved_path(path)
def find_actual_path(path):
"""
:param str|None path: Base path, if None or '.': look for first parent folder with a .git subfolder
:return str: Actual path to use as target
"""
if not path or path == ".":
path = git_parent_path(os.getcwd()) or "."
return runez.resolved_path(path)
def get_program_path(path=None):
if path is None:
path = runez.resolved_path(sys.argv[0])
if path.endswith(".py") or path.endswith(".pyc"):
packaged = runez.SYS_INFO.venv_bin_path(PICKLEY)
if runez.is_executable(packaged):
path = packaged # Convenience when running from debugger
return path
def _set_executable(self, path):
path = runez.resolved_path(path)
if runez.is_executable(path):
self.executable = path
if not self.major or not self.minor:
result = runez.run(self.executable, "--version", dryrun=False, fatal=False)
if result.succeeded:
m = RE_PYTHON_LOOSE.match(result.full_output)
if m:
self.major = m.group(3)
self.minor = m.group(4)
def test_capture(temp_folder, logged):
chatter = runez.resolved_path("chatter")
assert runez.write(chatter, CHATTER.strip(), fatal=False) == 1
assert runez.make_executable(chatter, fatal=False) == 1
assert runez.run(chatter, fatal=False) == "chatter"
r = runez.run(chatter, include_error=True, fatal=False)
assert r.startswith("chatter")
assert "No such file" in r
assert "Running: chatter" in logged
def _add_config_file(self, path, base=None):
path = runez.resolved_path(path, base=base)
if path and all(c.source != path
for c in self.configs) and os.path.exists(path):
values = runez.read_json(path, logger=LOG.warning)
if values:
self.configs.append(RawConfig(self, path, values))
included = values.get("include")
if included:
for additional in runez.flattened(included):
self._add_config_file(additional,
base=os.path.dirname(path))
def test_debian_mode(temp_folder, logged):
runez.write(
"foo/setup.py",
"import setuptools\nsetuptools.setup(name='foo', version='1.0')")
p = dummy_finalizer("root/apps")
assert p.dist == "root/apps/foo"
assert p.requirements == [runez.resolved_path("foo")]
assert "Using python:" in logged.pop()
# Symlink not created unless source effectively exists
p.symlink.apply("root/foo")
assert "skipping symlink" in logged.pop()
assert not os.path.isdir("root/usr/local/bin")
foo = runez.resolved_path("root/foo")
runez.touch(foo)
logged.pop()
# Simulate symlink
p.symlink.apply(foo)
assert "Symlinked root/usr/local/bin/foo -> root/foo" in logged.pop()
assert os.path.isdir("root/usr/local/bin")
assert os.path.islink("root/usr/local/bin/foo")
with patch("os.path.isdir", return_value=True): # pretend /apps exists
p = dummy_finalizer("root/apps")
assert "debian mode" in logged.pop()
assert p.dist == "/apps/foo"
with patch("runez.run",
return_value=runez.program.RunResult("usage: ...")):
assert p.validate_sanity_check(
"foo", "--version") == "does not respond to --version"
with patch("runez.run", return_value=runez.program.RunResult("failed")):
with pytest.raises(SystemExit):
p.validate_sanity_check("foo", "--version")
assert "'foo' failed --version sanity check" in logged.pop()
def __init__(self, project, dist, symlink):
"""
Args:
project (str): Folder where project to be packaged resides (must have a setup.py)
dist (str): Relative path to folder to use as 'dist' (where to deliver package)
symlink (str | None): Synlink specification, of the form 'root:root/...'
"""
self.folder = runez.resolved_path(project)
self.dist = dist
self.symlink = Symlinker(symlink) if symlink else None
self.sanity_check = None
self.requirements = []
self.compile = True
self.border = "reddit"
def _add_config(self, path, base=None):
"""
:param str path: Path to config file
:param str|None base: Base path to use to resolve relative paths (default: current working dir)
"""
path = runez.resolved_path(path, base=base)
if path not in self.config_paths:
settings_file = SettingsFile(self, path)
self.config_paths.append(path)
self.children.append(settings_file)
include = settings_file.include
if include:
for ipath in include:
self._add_config(ipath, base=settings_file.folder)
def main(ctx, debug, dryrun, base, index, config, python, delivery, packager):
"""
Package manager for python CLIs
"""
if any(opt in sys.argv
for opt in ctx.help_option_names): # pragma: no cover
return
if dryrun:
debug = True
if base:
base = runez.resolved_path(base)
if not os.path.exists(base):
sys.exit("Can't use %s as base: folder does not exist" %
short(base))
system.SETTINGS.set_base(base)
if not dryrun and ctx.invoked_subcommand in AUDITED:
file_location = system.SETTINGS.meta.full_path("audit.log")
else:
file_location = None
runez.log.setup(
debug=debug,
dryrun=dryrun,
greetings=":: {argv}",
console_format="%(levelname)s %(message)s" if debug else "%(message)s",
console_level=logging.WARNING,
console_stream=sys.stderr,
file_format=
"%(asctime)s %(timezone)s [%(process)d] %(context)s%(levelname)s - %(message)s",
file_level=logging.DEBUG,
file_location=file_location,
locations=None,
rotate="size:500k",
rotate_count=1,
)
runez.log.silence("pip")
# Disable logging.config, as pip tries to get smart and configure all logging...
logging.config.dictConfig = lambda x: None
system.SETTINGS.load_config(config=config,
delivery=delivery,
index=index,
packager=packager)
system.DESIRED_PYTHON = python
def package(build, dist, symlink, relocatable, sanity_check, folder):
"""
Package a project from source checkout
"""
build = runez.resolved_path(build)
dist = runez.resolved_path(dist)
folder = runez.resolved_path(folder)
system.SETTINGS.meta = meta_folder(build)
if not os.path.isdir(folder):
sys.exit("Folder %s does not exist" % short(folder))
setup_py = os.path.join(folder, "setup.py")
if not os.path.exists(setup_py):
sys.exit("No setup.py in %s" % short(folder))
with runez.CurrentFolder(folder):
# Some setup.py's assume their working folder is the folder where they're in
name = system.run_python(setup_py, "--name", fatal=False, dryrun=False)
if not name:
sys.exit("Could not determine package name from %s" %
short(setup_py))
runez.Anchored.add(folder)
p = PACKAGERS.resolved(name)
p.build_folder = build
p.dist_folder = dist
p.relocatable = relocatable
p.source_folder = folder
p.package()
p.create_symlinks(symlink)
p.sanity_check(sanity_check)
print("Packaged %s successfully, produced: %s" %
(short(folder), runez.represented_args(p.executables)))
runez.Anchored.pop(folder)
