def load_facts():
fact_fpath = ub.Path('~/misc/facts/facts.toml').expand()
with open(fact_fpath, 'r') as file:
fact_data = toml.load(file)
if 0:
with open(ub.Path('~/misc/facts/internal.toml').expand(), 'r') as file:
fact_data['facts'].extend(toml.load(file)['facts'])
return fact_data
def __init__(self, config):
self.config = config
self.repo_dpath = ub.Path(self.config['repodir'])
if self.config['repo_name'] is None:
self.config['repo_name'] = self.repo_dpath.name
self.repo_name = self.config['repo_name']
self._tmpdir = tempfile.TemporaryDirectory(prefix=self.repo_name)
self.template_infos = None
self.template_dpath = ub.Path('~/misc/templates/PYPKG').expand()
self.staging_dpath = ub.Path(self._tmpdir.name)
def remove_old_python2_headers():
import re
import ubelt as ub
from xdev import search_replace
from xdev import patterns
lines_to_remove = [
patterns.Pattern.from_regex(re.escape('from __future__ import absolute_import, ') + '.*', dotall=True),
patterns.Pattern.from_regex(re.escape('# -*- coding: utf-8 -*-') + '.*', dotall=True),
]
fpaths = set(ub.Path('~/code/watch/').expand().glob('**/*.py'))
fpaths = fpaths - {ub.Path('~/code/ubelt/dev/remove_ancient_constructs.py').expand()}
dry = 0
for fpath in fpaths:
# x = fpath.read_text().split('\n')[0:1][0]
for pat in lines_to_remove:
search_replace.sedfile(fpath, regexpr=pat, repl='', dry=dry, verbose=3)
def main(cls, cmdline=False, **kwargs):
from xdev.cli import docstr_stubgen
import ubelt as ub
config = cls(cmdline=cmdline, data=kwargs)
print(f'config={config}')
modname_or_path = config['module']
print(f'modname_or_path={modname_or_path}')
if modname_or_path is None:
raise ValueError('Must specify the module')
modpath = docstr_stubgen.modpath_coerce(modname_or_path)
modpath = ub.Path(modpath)
generated = docstr_stubgen.generate_typed_stubs(modpath)
for fpath, text in generated.items():
fpath = ub.Path(fpath)
print(f'Write fpath={fpath}')
fpath.write_text(text)
# Generate a py.typed file to mark the package as typed
if modpath.is_dir():
pytyped_fpath = (modpath / 'py.typed')
print(f'touch pytyped_fpath={pytyped_fpath}')
pytyped_fpath.touch()
def main():
import ubelt as ub
root_dpath = ub.Path('/etc')
import timerit
ti = timerit.Timerit(100, bestof=10, verbose=2)
for timer in ti.reset('time'):
with timer:
items1 = sorted(walk_with_scandir(root_dpath))
for timer in ti.reset('time'):
with timer:
items2 = sorted(walk_with_walk(root_dpath))
len(items1)
len(items2)
def render_facts():
"""
Render facts to a latex document
"""
import pylatex
from pylatex.base_classes.command import Options # NOQA
import pyqrcode
fact_data = load_facts()
class MDFramed(pylatex.base_classes.Environment):
_latex_name = 'mdframed'
packages = [pylatex.Package('mdframed')]
class SamePage(pylatex.base_classes.Environment):
_latex_name = 'samepage'
class ComposeContexts:
def __init__(self, *contexts):
self.contexts = contexts
def __enter__(self):
return [c.__enter__() for c in self.contexts]
def __exit__(self, a, b, c):
return [c.__exit__(a, b, c) for c in self.contexts[::-1]]
# class NewUnicodeChar(pylatex.base_classes.CommandBase):
# pass
# Dont use fontenc, lmodern, or textcomp
# https://tex.stackexchange.com/questions/179778/xelatex-under-ubuntu
doc = pylatex.Document('fact_document',
inputenc=None,
page_numbers=False,
indent=False,
fontenc=None,
lmodern=False,
textcomp=False)
doc.preamble.append(pylatex.Package('graphicx')) # For PNG images
# doc.preamble.append(pylatex.Package('svg', options=dict(inkscapearea='page')))
# doc.preamble.append(pylatex.Command('title', 'Facts'))
# doc.preamble.append(pylatex.Command('author', 'Anonymous author'))
# doc.preamble.append(pylatex.Command('date', pylatex.NoEscape(r'\today')))
# doc.append(pylatex.NoEscape(r'\maketitle'))
# doc.preamble.append(pylatex.Package('newunicodechar'))
# doc.preamble.append(pylatex.NoEscape(r'\newunicodechar{±}{$\pm$}'))
# doc.append(pylatex.NoEscape('13.787±0.020'))
# print(doc.dumps())
# doc.generate_pdf(clean_tex=False, compiler='xelatex')
# return
QR_REFERENCE = True
stop_flag = 0
image_dpath = ub.Path('~/misc/facts/images').expand().ensuredir()
# image_dpath =
for fact in ub.ProgIter(fact_data['facts']):
contexts = ComposeContexts(
# doc.create(SamePage()),
doc.create(MDFramed()),
doc.create(pylatex.MiniPage(width=r'0.99\textwidth')))
# with doc.create(pylatex.MiniPage(width=r'\textwidth')):
with contexts:
doc.append(pylatex.NoEscape(r'\paragraph{Fact:}'))
text = ub.paragraph(fact['text'])
if r'\[' in text:
found = list(
re.finditer(
'(' + re.escape(r'\[') + '|' + re.escape(r'\]') + ')',
text))
prev_x = 0
for a, b in ub.iter_window(found, step=2):
part = text[prev_x:a.span()[0]]
doc.append(part)
ax, bx = a.span()[1], b.span()[0]
part = pylatex.NoEscape(r'$' + text[ax:bx] + r'$ ')
doc.append(part)
prev_x = b.span()[1]
part = text[prev_x:]
doc.append(part)
else:
# if '$' in text:
# parts = text.split('$')
# for idx, p in enumerate(parts):
# if idx % 2 == 1:
# doc.append(pylatex.NoEscape('$' + p + '$ '))
# else:
# doc.append(p)
# else:
doc.append(text)
if QR_REFERENCE:
doc.append('\n')
num_refs = 0
for refline in fact['references'].split('\n'):
if refline.startswith('http'):
found = refline
image_fname = ub.hash_data(found,
base='abc')[0:16] + '.png'
image_fpath = image_dpath / image_fname
if not image_fpath.exists():
# pyqrcode.create(found).svg(fpath, scale=6)
pyqrcode.create(found).png(str(image_fpath),
scale=2)
doc.append(
pylatex.NoEscape(r'\includegraphics[width=90px]{' +
str(image_fpath) + '}'))
# doc.append(pylatex.NoEscape(r'\includesvg[width=120px]{' + fpath + '}'))
num_refs += 1
if num_refs > 3:
break
else:
doc.append(pylatex.NoEscape(r'\paragraph{References:}'))
with doc.create(pylatex.Itemize()) as itemize:
for refline in fact['references'].split('\n'):
if refline:
refline = refline.strip()
itemize.add_item(refline)
doc.append(pylatex.NoEscape(r'\bigskip'))
if stop_flag:
break
print(doc.dumps())
print('generate pdf')
doc.generate_pdf(str(ub.Path('~/misc/facts/fact_document').expand()),
clean_tex=True)
def benchmark_ubelt_import_time_robust():
import pandas as pd
import ubelt as ub
import kwplot
sns = kwplot.autosns(force='Qt5Agg')
prog = ub.codeblock(r'''
def _main():
import subprocess
import ubelt as ub
measurements = []
for i in range(200):
row = {}
# info = ub.cmd('python -X importtime -c "import ubelt"')
# text = info['err']
prog = subprocess.Popen('python -X importtime -c "import ubelt"', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
_, text = prog.communicate()
text = text.decode()
final_line = text.rstrip().split('\n')[-1]
partial = final_line.split(':')[1].split('|')
row['self_us'] = float(partial[0].strip())
row['cummulative'] = float(partial[1].strip())
measurements.append(row)
import pandas as pd
df = pd.DataFrame(measurements)
stats = pd.DataFrame({
'mean': df.mean(),
'std': df.std(),
'min': df.min(),
'max': df.max(),
'total': df.sum(),
})
info = stats.to_dict()
info['version'] = ub.__version__
print(info)
# print(stats)
_main()
''')
dpath = ub.Path(ub.ensure_app_cache_dir('ubelt/tests/test_version_import'))
fpath = dpath / 'do_test.py'
fpath.write_text(prog)
repo_root = ub.Path('$HOME/code/ubelt').expand()
info = ub.cmd('git tag', cwd=repo_root)
versions = [p for p in info['out'].split('\n') if p]
branches = ['dev/1.0.1', 'main'] + versions
fig = kwplot.figure(doclf=True)
ax = fig.gca()
bname_to_info = {}
rows = []
for bname in branches:
print('bname = {!r}'.format(bname))
ub.cmd('git checkout {}'.format(bname),
cwd=repo_root,
verbose=3,
check=True)
info = ub.cmd('python {}'.format(fpath), verbose=2)
dict_info = eval(info['out'])
bname_to_info[bname] = dict_info
for stat in ['mean', 'min', 'max']:
for type in ['self_us', 'cummulative']:
rows.append({
'version': dict_info['version'],
'stat': stat,
'type': type,
'time': dict_info[stat][type],
})
df = pd.DataFrame(rows[-1:])
print(df)
# ax.cla()
# sns.lineplot(data=df, x='version', y='time', hue='stat', style='type', ax=ax)
ub.cmd('git checkout {}'.format('dev/1.0.1'), cwd=repo_root)
df = pd.DataFrame(rows)
from distutils.version import LooseVersion
unique_versions = list(
map(str, sorted(map(LooseVersion, df['version'].unique()))))
df['release_index'] = df['version'].apply(
lambda x: unique_versions.index(x))
ax.cla()
kwplot.figure(fnum=2, pnum=(2, 1, 1), doclf=True)
ax = sns.lineplot(data=df[df['type'] == 'cummulative'],
x='release_index',
y='time',
hue='stat',
style='type',
marker='o')
ax.set_title('Ubelt import time over release history')
kwplot.figure(fnum=2, pnum=(2, 1, 2))
sns.lineplot(data=df[df['type'] == 'self_us'],
x='release_index',
y='time',
hue='stat',
style='type',
marker='o')
def multiple_items_from_a_dictionary():
"""
Spotlight:
ubelt.take
Modivation:
Working with Lists of Dictionaries
Requires:
kwimage
"""
...
"""
When working with data, a common pattern is to iterate through it, and
gather information about the work to be done, so a you can make a final
structured pass through the data.
In python we might do this by initializing an empty list and appending
**dictionary of information**, to the list. (or you might yield
dictionaries of information from a generator instead, either way you have a
flat list).
Some people might use lists of tuples instead of Lists of dictionaries, but
using dictionaries makes it easy to add new information later (and it works
very will with pandas).
"""
import ubelt as ub
import kwimage
kwimage_test_image_names = [
'airport', 'amazon', 'astro', 'carl', 'lowcontrast'
]
rows = []
for test_image in kwimage_test_image_names:
fpath = ub.Path(kwimage.grab_test_image_fpath(test_image))
imdata = kwimage.imread(fpath)
row = {
'mean': imdata.mean(),
'std': imdata.std(),
'sum': imdata.sum(),
'max': imdata.max(),
'min': imdata.min(),
}
rows.append(row)
"""
For each row, you might want to grab multiple specific items from it.
But having a separate assignment on each row wastes a lot of vertical
space.
"""
for row in rows:
mean = row['mean']
std = row['std']
sum = row['sum']
min = row['min']
max = row['max']
"""
You might put them one line explicitly, but that wastes a lot of horizontal
space
"""
for row in rows:
mean, std, sum, min, max = row['mean'], row['std'], row['sum'], row[
'min'], row['max']
"""
What if we try to be clever? We can use a list comprehension
"""
for row in rows:
mean, std, sum, min, max = [
row[k] for k in ['mean', 'std', 'sum', 'min', 'max']
]
"""
That's not too bad, but we can do better
"""
for row in rows:
mean, std, sum, min, max = ub.take(
row, ['mean', 'std', 'sum', 'min', 'max'])
def generate_typed_stubs(modpath):
"""
Attempt to use google-style docstrings, xdoctest, and mypy to generate
typed stub files.
Does not overwrite anything by itself.
Args:
modpath (PathLike): path to the module to generate types for
Returns:
Dict[PathLike, str]:
A dictionary mapping the path of each file to write to the text to
be written.
Notes:
FIXME: This currently requires my hacked version of mypy
Example:
>>> # xdoctest: +SKIP
>>> # xdoctest: +REQUIRES(module:mypy)
>>> # xdoctest: +REQUIRES(--hacked)
>>> from xdev.cli.docstr_stubgen import * # NOQA
>>> import xdev
>>> import ubelt as ub
>>> from xdev.cli import docstr_stubgen
>>> modpath = ub.Path(docstr_stubgen.__file__)
>>> generated = generate_typed_stubs(modpath)
>>> text = generated[ub.peek(generated.keys())]
>>> assert 'PathLike' in text
>>> assert 'Dict' in text
>>> print(text)
Ignore:
pyfile mypy.stubgen
# Delete compiled verisons so we can hack it
# ls $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm $VIRTUAL_ENV/lib/*/site-packages/mypy/*.so
# rm ~/.pyenv/versions/3.8.6/envs/pyenv3.8.6/lib/python3.8/site-packages/mypy/*.cpython-38-x86_64-linux-gnu.so
# This works I think?
if [[ ! -e "$HOME/code/mypy" ]]; then
git clone https://github.com/python/mypy.git $HOME/code/mypy
fi
(cd $HOME/code/mypy && git pull)
pip install -e $HOME/code/mypy
pip install MonkeyType
monkeytype run run_tests.py
monkeytype stub ubelt.util_dict
from typing import TypeVar
from mypy.applytype import get_target_type
z = TypeVar('Iterable')
get_target_type(z)
from mypy.expandtype import expand_type
expand_type(z, env={})
from mypy.types import get_proper_type
get_proper_type(z)
get_proper_type(dict)
import typing
get_proper_type(typing.Iterable)
from mypy.types import deserialize_type, UnboundType
import mypy.types as mypy_types
z = UnboundType('Iterable')
get_proper_type(dict)
from mypy.fastparse import parse_type_string
parse_type_string('dict', 'dict', 0, 0)
z = parse_type_string('typing.Iterator', 'Any', 0, 0)
get_proper_type(z)
"""
# import pathlib
# import ubelt as ub
import os
from mypy import stubgen
from mypy import defaults
from xdoctest import static_analysis
from os.path import join
import ubelt as ub
# modname = 'scriptconfig'
# module = ub.import_module_from_name(modname)
# modpath = ub.Path(module.__file__).parent
# for p in pathlib.Path(modpath).glob('*.pyi'):
# p.unlink()
modpath = ub.Path(modpath)
files = list(
static_analysis.package_modpaths(modpath,
recursive=True,
with_libs=1,
with_pkg=0))
# files = [f for f in files if 'deprecated' not in f]
# files = [join(ubelt_dpath, 'util_dict.py')]
if modpath.is_file():
output_dir = modpath.parent.parent
else:
output_dir = modpath.parent
options = stubgen.Options(pyversion=defaults.PYTHON3_VERSION,
no_import=True,
doc_dir='',
search_path=[],
interpreter=sys.executable,
ignore_errors=False,
parse_only=True,
include_private=False,
output_dir=output_dir,
modules=[],
packages=[],
files=files,
verbose=False,
quiet=False,
export_less=True)
# generate_stubs(options)
mypy_opts = stubgen.mypy_options(options)
py_modules, c_modules = stubgen.collect_build_targets(options, mypy_opts)
# Collect info from docs (if given):
sigs = class_sigs = None # type: Optional[Dict[str, str]]
if options.doc_dir:
sigs, class_sigs = stubgen.collect_docs_signatures(options.doc_dir)
# Use parsed sources to generate stubs for Python modules.
stubgen.generate_asts_for_modules(py_modules, options.parse_only,
mypy_opts, options.verbose)
generated = {}
for mod in py_modules:
assert mod.path is not None, "Not found module was not skipped"
target = mod.module.replace('.', '/')
if os.path.basename(mod.path) == '__init__.py':
target += '/__init__.pyi'
else:
target += '.pyi'
target = join(options.output_dir, target)
files.append(target)
with stubgen.generate_guarded(mod.module, target,
options.ignore_errors, options.verbose):
stubgen.generate_stub_from_ast(mod, target, options.parse_only,
options.pyversion,
options.include_private,
options.export_less)
gen = ExtendedStubGenerator(
mod.runtime_all,
pyversion=options.pyversion,
include_private=options.include_private,
analyzed=not options.parse_only,
export_less=options.export_less)
assert mod.ast is not None, "This function must be used only with analyzed modules"
mod.ast.accept(gen)
# print('gen.import_tracker.required_names = {!r}'.format(gen.import_tracker.required_names))
# print(gen.import_tracker.import_lines())
# print('mod.path = {!r}'.format(mod.path))
known_one_letter_types = {
# 'T', 'K', 'A', 'B', 'C', 'V',
'DT',
'KT',
'VT',
'T'
}
for type_var_name in set(gen.import_tracker.required_names) & set(
known_one_letter_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
custom_types = {'Hasher'}
for type_var_name in set(
gen.import_tracker.required_names) & set(custom_types):
gen.add_typing_import('TypeVar')
# gen.add_import_line('from typing import {}\n'.format('TypeVar'))
gen._output = [
'{} = TypeVar("{}")\n'.format(type_var_name, type_var_name)
] + gen._output
# Hack for specific module
# if mod.path.endswith('util_path.py'):
# gen.add_typing_import('TypeVar')
# # hack for variable inheritence
# gen._output = ['import pathlib\nimport os\n', "_PathBase = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath\n"] + gen._output
text = ''.join(gen.output())
text = postprocess_hacks(text, mod)
# Write output to file.
# subdir = ub.Path(target).parent
# if subdir and not os.path.isdir(subdir):
# os.makedirs(subdir)
generated[target] = text
# with open(target, 'w') as file:
# file.write(text)
return generated
def count_ubelt_usage():
config = UsageConfig(cmdline=True)
import ubelt as ub
import glob
from os.path import join
names = [
'xdoctest', 'netharn', 'xdev', 'xinspect', 'ndsampler',
'kwarray', 'kwimage', 'kwplot', 'kwcoco',
'scriptconfig', 'vimtk',
'mkinit', 'futures_actors', 'graphid',
'ibeis', 'plottool_ibeis', 'guitool_ibeis', 'utool', 'dtool_ibeis',
'vtool_ibeis', 'hesaff', 'torch_liberator', 'liberator',
] + config['extra_modnames']
code_repos = [ub.Path('~/code').expand() / name for name in names]
repo_dpaths = code_repos + [
# ub.Path('~/local').expand(),
ub.Path('~/misc').expand(),
]
all_fpaths = []
for repo_dpath in repo_dpaths:
name = repo_dpath.stem
fpaths = glob.glob(join(repo_dpath, '**', '*.py'), recursive=True)
for fpath in fpaths:
all_fpaths.append((name, fpath))
import re
pat = re.compile(r'\bub\.(?P<attr>[a-zA-Z_][A-Za-z_0-9]*)\b')
import ubelt as ub
pkg_to_hist = ub.ddict(lambda: ub.ddict(int))
for name, fpath in ub.ProgIter(all_fpaths):
with open(fpath, 'r') as file:
text = file.read()
for match in pat.finditer(text):
attr = match.groupdict()['attr']
if attr in ub.__all__:
pkg_to_hist[name][attr] += 1
hist_iter = iter(pkg_to_hist.values())
usage = next(hist_iter).copy()
for other in hist_iter:
for k, v in other.items():
usage[k] += v
for attr in ub.__all__:
usage[attr] += 0
for name in pkg_to_hist.keys():
pkg_to_hist[name] = ub.odict(sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1])
usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1])
if config['print_packages']:
print(ub.repr2(pkg_to_hist, nl=2))
if config['remove_zeros']:
for k, v in list(usage.items()):
if v == 0:
usage.pop(k)
if config['hardcoded_ubelt_hack']:
blocklist = [
'progiter', 'timerit', 'orderedset',
]
for k in list(usage):
if k in blocklist:
usage.pop(k, None)
elif k.startswith('util_'):
usage.pop(k, None)
elif k.startswith('_util_'):
usage.pop(k, None)
# ub._util_deprecated
from ubelt import _util_deprecated
if k in dir(_util_deprecated):
usage.pop(k, None)
print(ub.repr2(usage, nl=1))
return usage
def main(cls, cmdline=0, **kwargs):
"""
Ignore:
repodir = ub.Path('~/code/pyflann_ibeis').expand()
kwargs = {
'repodir': repodir,
'tags': ['binpy', 'erotemic', 'github'],
}
cmdline = 0
Example:
repodir = ub.Path.appdir('pypkg/demo/my_new_repo')
import sys, ubelt
sys.path.append(ubelt.expandpath('~/misc/templates/PYPKG'))
from apply_template import * # NOQA
kwargs = {
'repodir': repodir,
}
cmdline = 0
"""
import ubelt as ub
config = TemplateConfig(cmdline=cmdline, data=kwargs)
repo_dpath = ub.Path(config['repodir'])
repo_dpath.ensuredir()
IS_NEW_REPO = 0
create_new_repo_info = ub.codeblock('''
# TODO:
# At least instructions on how to create a new repo, or maybe an
# API call
https://github.com/new
git init
''')
print(create_new_repo_info)
if IS_NEW_REPO:
# TODO: git init
# TODO: github or gitlab register
pass
self = TemplateApplier(config)
self.setup().gather_tasks()
self.setup().apply()
if config['setup_secrets']:
setup_secrets_fpath = self.repo_dpath / 'dev/setup_secrets.sh'
if 'erotemic' in self.config['tags']:
environ_export = 'setup_package_environs_github_erotemic'
upload_secret_cmd = 'upload_github_secrets'
elif 'pyutils' in self.config['tags']:
environ_export = 'setup_package_environs_github_pyutils'
upload_secret_cmd = 'upload_github_secrets'
elif 'kitware' in self.config['tags']:
environ_export = 'setup_package_environs_gitlab_kitware'
upload_secret_cmd = 'upload_gitlab_repo_secrets'
else:
raise Exception
import cmd_queue
script = cmd_queue.Queue.create()
script.submit(
ub.codeblock(f'''
cd {self.repo_dpath}
source {setup_secrets_fpath}
{environ_export}
load_secrets
export_encrypted_code_signing_keys
git commit -am "Updated secrets"
{upload_secret_cmd}
'''))
script.rprint()