def represented_token(self, token):
linenum = token.start_mark.line + 1
column = token.start_mark.column + 1
result = "%s[%s,%s]" % (token.__class__.__name__, linenum, column)
value = getattr(token, "value", None)
if value is not None:
if token.id == "<scalar>":
value = represented_scalar(token.style, value)
elif token.id == "<anchor>":
value = "&%s" % value
elif token.id == "<alias>":
value = "*%s" % value
elif token.id == "<tag>":
assert isinstance(value, tuple)
value = " ".join(str(s) for s in runez.flattened(value))
elif token.id == "<directive>":
result += " %s" % token.name
value = " ".join(str(s) for s in runez.flattened(value))
else:
assert False
result = "%s %s" % (result, value)
return result
def package_specs(self, names=None, include_pickley=False):
"""
Args:
names (list | None): Package names, if empty: all installed
Returns:
(list[PackageSpec]): Corresponding PackageSpec-s
"""
if names:
names = runez.flattened(names, split=" ")
if include_pickley and PICKLEY not in names:
names.append(PICKLEY)
result = [self.resolved_bundle(name) for name in names]
result = runez.flattened(result, unique=True)
return [PackageSpec(self, name) for name in result]
result = []
if os.path.isdir(self.meta.path):
for fname in sorted(os.listdir(self.meta.path)):
if include_pickley or fname != PICKLEY:
fpath = os.path.join(self.meta.path, fname)
if os.path.isdir(fpath):
if os.path.exists(os.path.join(fpath,
".manifest.json")):
result.append(PackageSpec(self, fname))
return result
def flatten(self, key, split=None, direct=False):
if not self._contents:
return
node = self._contents.get(key)
if not node:
return
if direct:
self._contents[key] = runez.flattened(node,
split=split,
unique=True)
return
result = {}
for name, value in node.items():
result[name] = runez.flattened(value, split=split, unique=True)
self._contents[key] = result
def resolve(self):
if not os.path.isdir(self.folder):
abort("Folder %s does not exist" %
runez.red(runez.short(self.folder)))
req = self.requirements
if not req:
default_req = runez.resolved_path("requirements.txt",
base=self.folder)
if os.path.exists(default_req):
req = [default_req]
if req:
req = [("-r", runez.resolved_path(r, base=self.folder))
for r in req]
req = runez.flattened(req, shellify=True)
req.append(self.folder)
self.requirements = req
self.pspec = PackageSpec(CFG, self.folder)
LOG.info("Using python: %s" % self.pspec.python)
if self.dist.startswith("root/"):
# Special case: we're targeting 'root/...' probably for a debian, use target in that case to avoid venv relocation issues
target = self.dist[4:]
if os.path.isdir(target):
LOG.debug("debian mode: %s -> %s", self.dist, target)
self.dist = target
parts = self.dist.split("/")
if len(parts) <= 2:
# Auto-add package name to targets of the form root/subfolder (most typical case)
self.dist = os.path.join(self.dist, self.pspec.dashed)
def flatten(self, key, separator=None, direct=False):
if not self._contents:
return
node = self._contents.get(key)
if not node:
return
if direct:
self._contents[key] = runez.flattened(node,
split=(separator,
runez.UNIQUE))
return
result = {}
for name, value in node.items():
result[name] = runez.flattened(value,
split=(separator, runez.UNIQUE))
self._contents[key] = result
def find_python(self, pspec=None, fatal=True):
"""
Args:
pspec (PackageSpec | None): Package spec, when applicable
fatal (bool): If True, abort execution is no valid python could be found
Returns:
(runez.pyenv.PythonInstallation): Object representing python installation
"""
desired = self.get_value("python", pspec=pspec)
desired = runez.flattened(desired, split=",")
if not desired:
# Edge case: configured empty python... just use invoker in that case
return self.available_pythons.invoker
issues = []
python = None
for d in desired:
python = self.available_pythons.find_python(d)
if not python.problem:
return python
issues.append(
"Python '%s' skipped: %s" %
(runez.bold(runez.short(d)), runez.red(python.problem)))
for i in issues: # Warn only if no python could be found at all
LOG.warning(i)
if fatal:
abort("No suitable python installation found")
return python
def resolved_package_specs(names, auto_complete=False):
"""
:param list|tuple|str|None names: Names (possibly bundles) to resolve
:param bool auto_complete: If True, and no 'names' provided, return list of currently installed packages
:return list[PackageSpec]: Resolved specs
"""
result = []
if names:
if hasattr(names, "split"):
names = names.split()
for name in names:
if name.startswith("bundle:"):
bundle = SETTINGS.get_value("bundle.%s" % name[7:])
if bundle:
result.extend(bundle)
continue
result.append(name)
elif auto_complete and os.path.isdir(SETTINGS.meta.path):
for fname in sorted(os.listdir(SETTINGS.meta.path)):
if PackageSpec.is_valid(fname):
fpath = os.path.join(SETTINGS.meta.path, fname)
if os.path.isdir(fpath):
if os.path.exists(os.path.join(fpath, ".current.json")):
result.append(fname)
return [PackageSpec(name) for name in runez.flattened(result, unique=True)]
def _run_builtin_module(self, mod, *args, **kwargs):
args = runez.flattened(args, shellify=True)
python = self.python
if mod == "venv":
# Use original python installation when using the builtin venv module
python = self.venv_python.executable
return runez.run(python, "-m%s" % mod, *args, **kwargs)
def unbundled_names(self, names):
"""
:param list|tuple names: Names to unbundle
:return set: Resolved full set of names
"""
result = []
if names:
for name in names:
if name.startswith("bundle:"):
bundle = self.get_definition("bundle.%s" % name[7:])
if bundle and bundle.value:
result.extend(
runez.flattened(bundle.value,
split=(" ", runez.UNIQUE)))
continue
result.append(name)
return runez.flattened(result, split=(" ", runez.UNIQUE))
def _add_config_file(self, path, base=None):
path = runez.resolved_path(path, base=base)
if path and all(c.source != path
for c in self.configs) and os.path.exists(path):
values = runez.read_json(path, logger=LOG.warning)
if values:
self.configs.append(RawConfig(self, path, values))
included = values.get("include")
if included:
for additional in runez.flattened(included):
self._add_config_file(additional,
base=os.path.dirname(path))
def _expand_bundle(self, result, seen, bundle_name):
if not bundle_name or bundle_name in seen:
return
seen.add(bundle_name)
if not bundle_name.startswith("bundle:"):
result.append(bundle_name)
return
names = self.get_nested("bundle", bundle_name[7:])
if names:
for name in runez.flattened(names, split=" "):
self._expand_bundle(result, seen, name)
def _run_from_venv(self, command, *args, **kwargs):
"""
Should be called while holding the soft file lock in context only
:param str command: Command to run from that package (optionally specced with version)
:param args: Args to invoke program with
:param kwargs: Additional args
"""
cmd = system.PackageSpec(command)
if cmd.dashed in ("pip", "venv"):
return self._run_builtin_module(cmd.dashed, *args, **kwargs)
args = runez.flattened(args, shellify=True)
full_path = self._installed_module(cmd)
return runez.run(full_path, *args, **kwargs)
def _run_from_venv(self, package_name, *args, **kwargs):
"""
Should be called while holding the soft file lock in context only
:param str package_name: Pypi package to which command being ran belongs to
:param args: Args to invoke program with
:param kwargs: Additional args, use program= if entry point differs from 'package_name'
"""
if package_name == "pip":
return self._run_pip(*args, **kwargs)
args = runez.flattened(args, split=runez.SHELL)
program = kwargs.pop("program", package_name)
program, version = system.despecced(program)
full_path = self._installed_module(program, version=version)
return runez.run(full_path, *args, **kwargs)
def cmd_passthrough():
"""
Capture pass-through test
Run a program, capture its output as well as let it pass-through to stdout/stderr
"""
parser = runez.cli.parser()
args, unknown = parser.parse_known_args()
unknown = runez.flattened(unknown, split=" ")
if not unknown:
sys.exit("Provide command to run")
print("-- Running: %s\n" % unknown)
r = runez.run(*unknown, fatal=False, passthrough=True)
print("\n---- Captured: (exit code %s) ----" % r.exit_code)
print("\nstdout:\n%s" % (r.output or runez.dim("-empty-")))
print("\nstderr:\n%s" % (r.error or runez.dim("-empty-")))
def resolved_packages(self, names):
"""
:param list|tuple|str names: Names to resolve
:return set: Resolved names
"""
result = []
if names:
if hasattr(names, "split"):
names = names.split()
for name in names:
if name.startswith("bundle:"):
bundle = self.get_value("bundle.%s" % name[7:])
if bundle:
result.extend(bundle)
continue
result.append(name)
return runez.flattened(result, split=runez.UNIQUE)
def run(self, *args, **kwargs):
"""
Args:
*args: Command line args
**kwargs: If provided, format each arg with given `kwargs`
"""
if kwargs:
args = [runez.formatted(a, **kwargs) for a in args]
if len(args) == 1 and hasattr(args[0], "split"):
# Convenience: allow to provide full command as one string argument
args = args[0].split()
self.args = runez.flattened(args, split=runez.SHELL)
with IsolatedLogSetup(adjust_tmp=False):
with runez.CaptureOutput(dryrun=runez.DRYRUN) as logged:
self.logged = logged
runner = ClickWrapper.get_runner()
assert bool(self.main), "No main provided"
result = runner.invoke(self.main, args=self.args)
if result.output:
logged.stdout.buffer.write(result.output)
if result.exception and not isinstance(result.exception,
SystemExit):
try:
raise result.exception
except BaseException:
LOG.exception("Exited with stacktrace:")
self.exit_code = result.exit_code
if self.logged:
handler = WrappedHandler.find_wrapper()
if handler:
handler.reset()
title = runez.header("Captured output for: %s" %
runez.represented_args(self.args),
border="==")
LOG.info("\n%s\nmain: %s\nexit_code: %s\n%s\n", title, self.main,
self.exit_code, self.logged)
def cmd_colors():
"""Show a coloring sample"""
parser = runez.cli.parser()
parser.add_argument("--border",
choices=NAMED_BORDERS,
help="Use custom border.")
parser.add_argument("--color",
action="store_true",
help="Use colors (on by default on ttys).")
parser.add_argument("--no-color",
action="store_true",
help="Do not use colors (even if on tty).")
parser.add_argument(
"--bg",
help="Show bg variant(s) (comma-separated list of color names).")
parser.add_argument("--flavor",
help="Show specific flavor (neutral, light or dark).")
args = parser.parse_args()
enable_colors = None
if args.no_color:
enable_colors = False
elif args.color:
enable_colors = True
with runez.ActivateColors(enable=enable_colors, flavor=args.flavor):
print("Backend: %s" % runez.color.backend)
_show_fgcolors(border=args.border)
if args.bg:
for name in runez.flattened(args.bg, split=","):
color = runez.color.bg.get(name)
if color is None:
print("Unknown bg color '%s'" % name)
else:
_show_fgcolors(bg=color, border=args.border)
def get_samples(cls, sample_name):
result = []
for name in runez.flattened([sample_name], split=","):
result.extend(cls.scan_samples(name))
return sorted(result, key=lambda x: x.key)
def test_flattened():
assert runez.flattened(None) == []
assert runez.flattened([None]) == []
assert runez.flattened([None], keep_empty=True) == [None]
assert runez.flattened(["1"], "2", transform=int) == [1, 2]
assert runez.flattened(None, [runez.UNSET, 0]) == [0]
assert runez.flattened(None, [runez.UNSET, 0], shellify=True) == ["0"]
assert runez.flattened(None, [runez.UNSET, 0], keep_empty=None) == []
assert runez.flattened(None, [runez.UNSET, 0], keep_empty=False) == [0]
assert runez.flattened(None, [runez.UNSET, 0],
keep_empty=True) == [None, 0]
assert runez.flattened(None, [runez.UNSET, 0], keep_empty="") == ["", 0]
assert runez.flattened(None, [runez.UNSET, 0],
keep_empty="null") == ["null", 0]
assert runez.flattened(None, [runez.UNSET, 0],
None,
keep_empty="null",
unique=False) == ["null", 0, "null"]
assert runez.flattened(None, [runez.UNSET, 0],
None,
keep_empty="null",
unique=True) == ["null", 0]
assert runez.flattened(None, [runez.UNSET, 0],
keep_empty="",
shellify=True) == ["", "0"]
assert runez.flattened(None, None, keep_empty=False, unique=True) == []
assert runez.flattened(None, None, shellify=True) == []
assert runez.flattened(None,
None,
runez.UNSET,
None,
runez.UNSET,
unique=True) == []
assert runez.flattened(None,
None,
runez.UNSET,
None,
runez.UNSET,
keep_empty=True,
unique=True) == [None]
assert runez.flattened(None, None, keep_empty=None) == []
assert runez.flattened(None, None, keep_empty="") == ["", ""]
assert runez.flattened(None, None, keep_empty="null") == ["null", "null"]
assert runez.flattened(None, None, keep_empty="null",
unique=True) == ["null"]
assert runez.flattened(["-a", [None, "b", runez.UNSET],
runez.UNSET]) == ["-a", "b"]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET],
unique=True) == ["-a", "b"]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET],
keep_empty=True) == ["-a", None, "b"]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET],
keep_empty="") == ["-a", "", "b"]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET],
shellify=True) == ["b"]
assert runez.flattened(
["-a", [runez.UNSET, "b", runez.UNSET], runez.UNSET],
shellify=True) == ["b"]
assert runez.flattened(["a b"]) == ["a b"]
assert runez.flattened([["a b"]]) == ["a b"]
assert runez.flattened(["-r", None, "foo"]) == ["-r", "foo"]
assert runez.flattened(["-r", None, "foo"],
keep_empty=True) == ["-r", None, "foo"]
assert runez.flattened(["foo", "-r", None, "bar"],
shellify=True) == ["foo", "bar"]
assert runez.flattened(["-r", None, "foo"], unique=True) == ["-r", "foo"]
assert runez.flattened(["-r", None, "foo"], keep_empty=True,
unique=True) == ["-r", None, "foo"]
# Sanitized
assert runez.flattened(("a", None, ["b", None]), unique=True) == ["a", "b"]
assert runez.flattened(("a", None, ["b", None]),
keep_empty=True,
unique=True) == ["a", None, "b"]
# Shell cases
assert runez.flattened([None, "a", "-f", "b", "c", None],
shellify=True) == ["a", "-f", "b", "c"]
assert runez.flattened(["a", "-f", "b", "c"],
shellify=True) == ["a", "-f", "b", "c"]
assert runez.flattened([None, "-f", "b", None],
shellify=True) == ["-f", "b"]
assert runez.flattened(["a", "-f", None, "c"], shellify=True) == ["a", "c"]
# Verify -flag gets removed with shellify
assert runez.flattened(["a", "-f", None, "c"], shellify=True) == ["a", "c"]
assert runez.flattened(["a", "-f", None, "c"],
keep_empty=False,
shellify=True) == ["a", "c"]
assert runez.flattened(["a", "-f", None, "c"],
keep_empty=True,
shellify=True) == ["a", "c"]
# shellify influences keep_empty
expected = ["a", "-f", "", "c", "0", ""]
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
shellify=True) == expected
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
keep_empty=False,
shellify=True) == expected
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
keep_empty=True,
shellify=True) == expected
# Override keep_empty
assert runez.flattened(["a", "-f", None, "c"],
keep_empty=None,
shellify=True) == ["a", "c"]
assert runez.flattened(["a", "-f", None, "c"],
keep_empty="",
shellify=True) == ["a", "-f", "", "c"]
assert runez.flattened(["a", "-f", None, "c"],
keep_empty="null",
shellify=True) == ["a", "-f", "null", "c"]
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
keep_empty=None,
shellify=True) == ["a", "c"]
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
keep_empty="",
shellify=True) == ["a", "-f", "", "c", "", "0", ""]
assert runez.flattened(["a", "-f", "", "c", None, 0, ""],
keep_empty="null",
shellify=True) == [
"a", "-f", "", "c", "null", "0", ""
]
# keep_empty with transform
def keep_odds(i):
if i % 2:
return i
return 0 if i % 4 else None
sample = list(range(8))
assert runez.flattened(sample, transform=keep_odds,
keep_empty=True) == [None, 1, 0, 3, None, 5, 0, 7]
assert runez.flattened(sample, transform=keep_odds,
keep_empty=False) == [1, 0, 3, 5, 0, 7]
assert runez.flattened(sample, transform=keep_odds,
keep_empty=None) == [1, 3, 5, 7]
assert runez.flattened(sample, transform=keep_odds, keep_empty="foo") == [
"foo", 1, 0, 3, "foo", 5, 0, 7
]
def _run_pip(self, *args, **kwargs):
args = runez.flattened(args, split=runez.SHELL)
return runez.run(self.pip, *args, **kwargs)
def cmd_import_speed():
"""Show average import time of top-level python packages installed in this venv"""
parser = runez.cli.parser()
parser.add_argument("--all", action="store_true", help="Show all.")
parser.add_argument("--border",
choices=NAMED_BORDERS,
default="reddit",
help="Use custom border.")
parser.add_argument("--iterations",
"-i",
type=int,
default=3,
help="Number of measurements to average.")
parser.add_argument("name",
nargs="*",
help="Names of modules to show (by default: all).")
args = parser.parse_args()
names = runez.flattened(args.name, split=",")
if args.all:
names.extend(_all_deps())
if not names:
sys.exit("Please specify module names, or use --all")
names = sorted(set(names))
times = []
fastest = None
slowest = None
for name in names:
t = ImportTime(name, iterations=args.iterations)
times.append(t)
if t.cumulative is None:
continue
if fastest is None or (t.cumulative < fastest.cumulative):
fastest = t
if slowest is None or t.cumulative > slowest.cumulative:
slowest = t
table = PrettyTable("Module,-X cumulative,Elapsed,Vs fastest,Note",
border=args.border)
table.header[3].align = "center"
mid = _get_mid(times) or 0
for t in sorted(times):
if t.cumulative is None:
c = e = f = None
else:
factor = t.elapsed / fastest.elapsed
c = runez.represented_duration(t.cumulative / 1000000, span=-2)
e = runez.represented_duration(t.elapsed, span=-2)
f = "x%.2f" % factor
if t is fastest:
f = ""
elif t is slowest:
f = runez.red(f)
elif t.elapsed and t.elapsed > mid:
f = runez.orange(f)
table.add_row(t.module_name, c, e, f, t.problem or "")
print(table)
def test_flattened():
assert runez.flattened(None) == [None]
assert runez.flattened([None]) == [None]
assert runez.flattened(None, split=runez.SANITIZED) == []
assert runez.flattened(None, split=runez.SHELL) == []
assert runez.flattened(None, split=runez.UNIQUE) == [None]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET]) == ["-a", None, "b", runez.UNSET, runez.UNSET]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET], split=runez.UNIQUE) == ["-a", None, "b", runez.UNSET]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET], split=runez.SANITIZED) == ["-a", "b"]
assert runez.flattened(["-a", [None, "b", runez.UNSET], runez.UNSET], split=runez.SHELL) == ["b"]
assert runez.flattened(["-a", [runez.UNSET, "b", runez.UNSET], runez.UNSET], split=runez.SHELL) == ["b"]
assert runez.flattened(["a b"]) == ["a b"]
assert runez.flattened([["a b"]]) == ["a b"]
assert runez.flattened(["-r", None, "foo"]) == ["-r", None, "foo"]
assert runez.flattened(["-r", None, "foo"], split=runez.SANITIZED) == ["-r", "foo"]
assert runez.flattened(["-r", None, "foo"], split=runez.SHELL) == ["foo"]
assert runez.flattened(["-r", None, "foo"], split=runez.UNIQUE) == ["-r", None, "foo"]
assert runez.flattened(["-r", None, "foo"], split=runez.SANITIZED | runez.UNIQUE) == ["-r", "foo"]
# Sanitized
assert runez.flattened(("a", None, ["b", None]), split=runez.UNIQUE) == ["a", None, "b"]
assert runez.flattened(("a", None, ["b", None]), split=runez.SANITIZED | runez.UNIQUE) == ["a", "b"]
# Shell cases
assert runez.flattened([None, "a", "-f", "b", "c", None], split=runez.SHELL) == ["a", "-f", "b", "c"]
assert runez.flattened(["a", "-f", "b", "c"], split=runez.SHELL) == ["a", "-f", "b", "c"]
assert runez.flattened([None, "-f", "b", None], split=runez.SHELL) == ["-f", "b"]
assert runez.flattened(["a", "-f", None, "c"], split=runez.SHELL) == ["a", "c"]
# Splitting on separator
assert runez.flattened("a b b") == ["a b b"]
assert runez.flattened("a b b", split=" ") == ["a", "b", "b"]
assert runez.flattened("a b b", split=(" ", runez.UNIQUE)) == ["a", "b"]
assert runez.flattened("a b b", split=(None, runez.UNIQUE)) == ["a b b"]
assert runez.flattened("a b b", split=("", runez.UNIQUE)) == ["a b b"]
assert runez.flattened("a b b", split=("+", runez.UNIQUE)) == ["a b b"]
# Unique
assert runez.flattened(["a", ["a", ["b", ["b", "c"]]]]) == ["a", "a", "b", "b", "c"]
assert runez.flattened(["a", ["a", ["b", ["b", "c"]]]], split=runez.UNIQUE) == ["a", "b", "c"]
assert runez.flattened(["a b", None, ["a b c"], "a"], split=runez.UNIQUE) == ["a b", None, "a b c", "a"]
assert runez.flattened(["a b", None, ["a b c"], "a"], split=(" ", runez.UNIQUE)) == ["a", "b", None, "c"]
assert runez.flattened(["a b", None, ["a b c"], "a"], split=(" ", runez.SANITIZED | runez.UNIQUE)) == ["a", "b", "c"]
assert runez.represented_args(None) == ""
assert runez.represented_args([]) == ""
assert runez.represented_args([1, 2], separator="+") == "1+2"
def test_flattened_split():
# Splitting on a given char
assert runez.flattened("a b b") == ["a b b"]
assert runez.flattened("a b\n b", split=" ") == ["a", "b", "b"]
assert runez.flattened("a b\n b", split=True) == ["a", "b", "b"]
assert runez.flattened("a b\n \n \n b", split=" ",
unique=True) == ["a", "b"]
assert runez.flattened("a b b", unique=True) == ["a b b"]
assert runez.flattened("a b b", split="", unique=True) == ["a b b"]
assert runez.flattened("a b b", split="+", unique=True) == ["a b b"]
assert runez.flattened("a,,b", "c", split=",") == ["a", "b", "c"]
# Unique
assert runez.flattened(["a", ["a", ["b",
["b",
"c"]]]]) == ["a", "a", "b", "b", "c"]
assert runez.flattened(["a", ["a", ["b", ["b", "c"]]]],
unique=True) == ["a", "b", "c"]
assert runez.flattened(["a b", None, ["a b c"], "a"],
unique=True) == ["a b", "a b c", "a"]
assert runez.flattened(["a b", None, ["a b c"], "a"],
keep_empty=True,
unique=True) == ["a b", None, "a b c", "a"]
assert runez.flattened(["a b", None, ["a b c"], "a"],
split=" ",
unique=True) == ["a", "b", "c"]
assert runez.flattened(["a b", None, ["a b c"], "a"],
split=" ",
keep_empty=False,
unique=True) == ["a", "b", "c"]
def name_available(target, name):
"""Check whether a name is taken or not (.com, .org, .dev and github"""
import datetime
import socket
import requests
now = datetime.datetime.now()
now = now.strftime("%Y-%m-%d")
names = runez.flattened(name)
domains = "com org dev io net".split()
k_github = "github"
k_pypi = "pypi"
k_available = "----"
k_taken = "taken"
report = {}
existing = {}
has_header = False
if os.path.exists(target):
with open(target) as fh:
for line in fh:
if line.startswith("Name"):
has_header = True
continue
line = line.rstrip("\n")
name, _, _ = line.partition("\t")
name = name.strip()
existing[name] = line
if names == ["_redo"]:
has_header = False
names = sorted(existing)
existing = {}
for name in names:
if name in existing:
print("Skipping '%s': already queried %s" % (name, existing[name]))
continue
report[name] = "%-8s\t%-10s" % (name, now)
r = requests.head("https://github.com/%s" % name)
report[name] += "\t%s" % (k_available
if r.status_code >= 400 else k_taken)
r = requests.head("https://pypi.org/project/%s/" % name)
report[name] += "\t%s" % (k_available
if r.status_code >= 400 else k_taken)
for domain in domains:
hostname = "%s.%s" % (name, domain)
try:
socket.gethostbyname_ex(hostname)
status = k_taken
except socket.gaierror:
status = k_available
report[name] += "\t%-4s" % status
header = ["%-8s" % "Name", "%-10s" % "Checked", k_github, k_pypi]
header += [".%-4s" % s for s in domains]
header = "\t".join(header)
header = "%s\n" % header.strip()
if not has_header:
with open(target, "w") as fh:
fh.write(header)
if report:
print(header.strip())
with open(target, "a") as fh:
for name, line in sorted(report.items()):
fh.write(line)
fh.write("\n")
print(line)
print("")
print("%s names added" % len(report))
for dist in ws:
if _is_interesting_dist(dist.key):
top_level = _find_top_level(base, dist)
if top_level:
result.append(top_level)
return result
# Usual dev libs that are not interesting for --all import times, they import ultra fast...
# They can always be stated as argument explicitly to show their import times anyway
DEV_LIBS = """
attrs coverage more-itertools packaging pip pluggy py pyparsing python-dateutil setuptools six wcwidth wheel zipp
binaryornot cookiecutter click future
"""
DEV_LIBS = set(runez.flattened(DEV_LIBS, split=" "))
def _is_interesting_dist(key):
if key.startswith("pytest") or key.startswith("importlib"):
return False
return key not in DEV_LIBS
def _find_top_level(base, dist):
name = dist.key.replace("-", "_").replace(".", "_")
top_level = os.path.join(base, "%s-%s.dist-info" % (name, dist.version),
"top_level.txt")
for line in runez.readlines(top_level):
if not line.startswith("_") and line: