def main():
modnames = [
'scriptconfig',
'kwarray',
'kwimage',
'kwplot',
'ndsampler',
'netharn',
]
print('--- force module side effects --- ')
for modname in modnames:
ub.import_module_from_name(modname)
print('--- begin module query ---')
for modname in modnames:
info = query_module_pypi_info(modname)
print(ub.repr2(info))
if info['local_version'] > info['pypi_version']:
print('--------')
print("NEED TO PUBLISH {}".format(modname))
print('https://travis-ci.org/Erotemic/{}'.format(modname))
print('--------')
def main():
modnames = [
'utool',
'vtool_ibeis',
'guitool_ibeis',
'plottool_ibeis',
'pyhesaff',
'pyflann_ibeis',
'ibeis',
]
print('--- force module side effects --- ')
for modname in modnames:
ub.import_module_from_name(modname)
print('--- begin module query ---')
for modname in modnames:
info = query_module_pypi_info(modname)
print(ub.repr2(info))
if info['local_version'] > info['pypi_version']:
print('--------')
print("NEED TO PUBLISH {}".format(modname))
print('https://travis-ci.org/Erotemic/{}'.format(modname))
print('--------')
def gen_api_for_docs():
"""
import sys, ubelt
sys.path.append(ubelt.expandpath('~/code/ubelt/dev'))
from gen_api_for_docs import * # NOQA
"""
from count_usage_freq import count_usage
config, usage = count_usage(modname='kwimage')
modname = config['modname']
gaurd = ('=' * 64 + ' ' + '=' * 16)
print(gaurd)
print('{:<64} {:>8}'.format(' Function name ', 'Usefulness'))
print(gaurd)
for key, value in usage.items():
print('{:<64} {:>16}'.format(':func:`' + modname + '.' + key + '`',
value))
print(gaurd)
import ubelt as ub
module = ub.import_module_from_name(modname)
attrnames = module.__all__
if hasattr(module, '__protected__'):
# Hack for lazy imports
for subattr in module.__protected__:
submod = ub.import_module_from_name(modname + '.' + subattr)
setattr(module, subattr, submod)
attrnames += module.__protected__
for attrname in attrnames:
member = getattr(module, attrname)
submembers = getattr(member, '__all__', None)
# if attrname.startswith('util_'):
if not submembers:
from mkinit.static_mkinit import _extract_attributes
try:
submembers = _extract_attributes(member.__file__)
except AttributeError:
pass
if submembers:
print('\n:mod:`{}.{}`'.format(modname, attrname))
print('-------------')
for subname in submembers:
if not subname.startswith('_'):
print(':func:`{}.{}`'.format(modname, subname))
submembers = dir(member)
print('config = ' + ub.repr2(config.asdict(), nl=1))
def code(self):
if self._code is None:
try:
if self._expanded or self.type == 'Assign':
# always use astunparse if we have expanded
raise Exception
# Attempt to dynamically extract the source code because it
# keeps formatting better.
module = ub.import_module_from_name(self.modname)
obj = getattr(module, self.name)
self._code = inspect.getsource(obj).strip('\n')
except Exception:
# Fallback on static sourcecode extraction
# (NOTE: it should be possible to keep formatting with a bit of
# work)
self._code = unparse(self.node).strip('\n')
return self._code
def query_module_pypi_info(modname, verbose=0):
"""
Determine the lastest version of a module on pypi and the current installed
version.
"""
cmdinfo = ub.cmd('yolk -V {}'.format(modname), verbose=verbose, check=True)
pypi_version = LooseVersion(cmdinfo['out'].strip().split(' ')[1])
try:
module = ub.import_module_from_name(modname)
except ImportError:
local_version = None
else:
local_version = LooseVersion(module.__version__)
info = {
'modname': modname,
'pypi_version': pypi_version,
'local_version': local_version,
}
return info
"""
for modname in ut.ProgIter(
AUTOLOAD_PLUGIN_MODNAMES,
'loading plugins',
enabled=ut.VERYVERBOSE,
adjust=False,
freq=1,
):
if isinstance(modname, tuple):
flag, modname = modname
if ut.get_argflag(flag):
continue
try:
# ut.import_modname(modname)
ub.import_module_from_name(modname)
except ImportError:
if 'wbia_cnn' in modname:
import warnings
warnings.warn('Unable to load plugin: {!r}'.format(modname))
else:
raise
# NOTE: new plugin code needs to be hacked in here currently
# this is not a long term solution. THE Long term solution is to get these
# working (which are partially integrated)
# python -m wbia dev_autogen_explicit_imports
# python -m wbia dev_autogen_explicit_injects
# Ensure that all injectable modules are imported before constructing the
def _lookup_deprecated_attribute(key):
import ubelt as ub
# mapping from module name to the attributes that were moved there.
refactored = {
'kwarray': [
'ArrayAPI', 'DataFrameArray', 'DataFrameLight', 'FlatIndexer',
'LocLight', 'RunningStats', 'apply_grouping', 'arglexmax',
'argmaxima', 'argminima', 'atleast_nd', 'boolmask', 'ensure_rng',
'group_consecutive', 'group_consecutive_indices', 'group_indices',
'group_items', 'isect_flags', 'iter_reduce_ufunc',
'maxvalue_assignment', 'mincost_assignment', 'mindist_assignment',
'one_hot_embedding', 'one_hot_lookup', 'random_combinations',
'random_product', 'seed_global', 'setcover', 'shuffle',
'standard_normal', 'standard_normal32', 'standard_normal64',
'stats_dict', 'uniform', 'uniform32'
],
'kwimage': [
'BASE_COLORS',
'Boxes',
'CSS4_COLORS',
'Color',
'Coords',
'Detections',
'Heatmap',
'Mask',
'MaskList',
'MultiPolygon',
'Points',
'PointsList',
'Polygon',
'PolygonList',
'Segmentation',
'SegmentationList',
'TABLEAU_COLORS',
'TORCH_GRID_SAMPLE_HAS_ALIGN',
'XKCD_COLORS',
'add_homog',
'atleast_3channels',
'available_nms_impls',
'convert_colorspace',
'daq_spatial_nms',
'decode_run_length',
'draw_boxes_on_image',
'draw_clf_on_image',
'draw_line_segments_on_image',
'draw_text_on_image',
'draw_vector_field',
'encode_run_length',
'ensure_alpha_channel',
'ensure_float01',
'ensure_uint255',
'fourier_mask',
'gaussian_patch',
'grab_test_image',
'grab_test_image_fpath',
'imread',
'imresize',
'imscale',
'imwrite',
'load_image_shape',
'make_channels_comparable',
'make_heatmask',
'make_orimask',
'make_vector_field',
'non_max_supression',
'normalize',
'num_channels',
'overlay_alpha_images',
'overlay_alpha_layers',
'radial_fourier_mask',
'remove_homog',
'rle_translate',
'smooth_prob',
'stack_images',
'stack_images_grid',
'subpixel_accum',
'subpixel_align',
'subpixel_getvalue',
'subpixel_maximum',
'subpixel_minimum',
'subpixel_set',
'subpixel_setvalue',
'subpixel_slice',
'subpixel_translate',
'warp_image',
'warp_points',
'warp_tensor',
],
'kwplot': [
'BackendContext',
'Color',
'PlotNums',
'autompl',
'autoplt',
'distinct_colors',
'distinct_markers',
'draw_boxes',
'draw_boxes_on_image',
'draw_clf_on_image',
'draw_line_segments',
'draw_points',
'draw_text_on_image',
'ensure_fnum',
'figure',
'imshow',
'legend',
'make_conv_images',
'make_heatmask',
'make_legend_img',
'make_orimask',
'make_vector_field',
'multi_plot',
'next_fnum',
'plot_convolutional_features',
'plot_matrix',
'plot_surface3d',
'set_figtitle',
'set_mpl_backend',
'show_if_requested',
],
'ubelt': [
'CacheStamp',
]
}
ERROR_ON_ACCESS = True
for modname, attrs in refactored.items():
if key in attrs:
text = ub.paragraph('''
The attribute `netharn.util.{key}` is deprecated.
It was refactored and moved to `{modname}.{key}`.
''').format(key=key, modname=modname)
if ERROR_ON_ACCESS:
raise AttributeError(text)
else:
module = ub.import_module_from_name(modname)
import warnings
warnings.warn(text)
return getattr(module, key)
if key in ['SlidingIndexDataset', 'SlidingSlices']:
raise AttributeError('Deprecated {}, but still available in '
'netharn.util.util_slider_dep'.format(key))
raise AttributeError(key)
def test_import_modname_builtin():
module = ub.import_module_from_name('ast')
import ast
assert module is ast
def count_usage(cmdline=True):
config = UsageConfig(cmdline=cmdline)
import ubelt as ub
import glob
from os.path import join
names = [
'netharn',
'ndsampler',
'kwimage',
'kwplot',
] + config['extra_modnames']
all_fpaths = []
for name in names:
if name:
repo_fpath = ub.expandpath(join('~/code', name))
fpaths = glob.glob(join(repo_fpath, '**', '*.py'), recursive=True)
for fpath in fpaths:
all_fpaths.append((name, fpath))
print('names = {}'.format(ub.repr2(names)))
import re
import ubelt as ub
modname = 'kwarray'
module = ub.import_module_from_name(modname)
package_name = module.__name__
package_allvar = module.__all__
pat = re.compile(r'\b' + package_name +
r'\.(?P<attr>[a-zA-Z_][A-Za-z_0-9]*)\b')
pkg_to_hist = ub.ddict(lambda: ub.ddict(int))
for name, fpath in ub.ProgIter(all_fpaths):
# print('fpath = {!r}'.format(fpath))
text = ub.readfrom(fpath, verbose=0)
# text = open(fpath, 'r').read()
for match in pat.finditer(text):
attr = match.groupdict()['attr']
if attr in package_allvar:
pkg_to_hist[name][attr] += 1
hist_iter = iter(pkg_to_hist.values())
usage = next(hist_iter).copy()
for other in hist_iter:
for k, v in other.items():
usage[k] += v
for attr in package_allvar:
usage[attr] += 0
for name in pkg_to_hist.keys():
pkg_to_hist[name] = ub.odict(
sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1])
usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1])
if config['print_packages']:
print(ub.repr2(pkg_to_hist, nl=2))
if config['remove_zeros']:
for k, v in list(usage.items()):
if v == 0:
usage.pop(k)
# if config['hardcoded_ubelt_hack']:
# for k in list(usage):
# if k.startswith('util_'):
# usage.pop(k)
# if k.startswith('_util_'):
# usage.pop(k)
# # ub._util_deprecated
# from ubelt import _util_deprecated
# if k in dir(_util_deprecated):
# usage.pop(k)
print(ub.repr2(usage, nl=1))
return usage
def visit_func_def(self,
o: FuncDef,
is_abstract: bool = False,
is_overload: bool = False) -> None:
import ubelt as ub
if (self.is_private_name(o.name, o.fullname)
or self.is_not_in_all(o.name)
or (self.is_recorded_name(o.name) and not is_overload)):
self.clear_decorators()
return
if not self._indent and self._state not in (
EMPTY, FUNC) and not o.is_awaitable_coroutine:
self.add('\n')
if not self.is_top_level():
self_inits = find_self_initializers(o)
for init, value in self_inits:
if init in self.method_names:
# Can't have both an attribute and a method/property with the same name.
continue
init_code = self.get_init(init, value)
if init_code:
self.add(init_code)
# dump decorators, just before "def ..."
for s in self._decorators:
self.add(s)
self.clear_decorators()
self.add(
"%s%sdef %s(" %
(self._indent, 'async ' if o.is_coroutine else '', o.name))
self.record_name(o.name)
# import ubelt as ub
# if o.name == 'dzip':
# import xdev
# xdev.embed()
def _hack_for_info(info):
type_name = info['type']
if type_name is not None:
results = hacked_typing_info(type_name)
for typing_arg in results['typing_imports']:
self.add_typing_import(typing_arg)
for line in results['import_lines']:
self.add_import_line(line)
info['type'] = results['type_name']
name_to_parsed_docstr_info = {}
return_parsed_docstr_info = None
fullname = o.name
if getattr(self, '_IN_CLASS', None) is not None:
fullname = self._IN_CLASS + '.' + o.name
curr = ub.import_module_from_name(self.module)
# curr = sys.modules.get(self.module)
# print('o.name = {!r}'.format(o.name))
# print('fullname = {!r}'.format(fullname))
for part in fullname.split('.'):
# print('part = {!r}'.format(part))
# print('curr = {!r}'.format(curr))
curr = getattr(curr, part, None)
# print('curr = {!r}'.format(curr))
real_func = curr
# print('real_func = {!r}'.format(real_func))
# if o.name == 'dict_union':
# import xdev
# xdev.embed()
if real_func is not None and real_func.__doc__ is not None:
from mypy import fastparse
from xdoctest.docstr import docscrape_google
parsed_args = None
# parsed_ret = None
blocks = docscrape_google.split_google_docblocks(
real_func.__doc__)
# print('blocks = {}'.format(ub.repr2(blocks, nl=1)))
for key, block in blocks:
# print(f'key={key}')
lines = block[0]
if key == 'Returns':
# print(f'lines={lines}')
for retdict in docscrape_google.parse_google_retblock(
lines):
# print(f'retdict={retdict}')
_hack_for_info(retdict)
return_parsed_docstr_info = (key, retdict['type'])
if return_parsed_docstr_info is None:
print(
'Warning: return block for {} might be malformed'
.format(real_func))
if key == 'Yields':
for retdict in docscrape_google.parse_google_retblock(
lines):
_hack_for_info(retdict)
return_parsed_docstr_info = (key, retdict['type'])
if return_parsed_docstr_info is None:
print(
'Warning: return block for {} might be malformed'
.format(real_func))
if key == 'Args':
# hack for *args
lines = '\n'.join(
[line.lstrip('*') for line in lines.split('\n')])
# print('lines = {!r}'.format(lines))
parsed_args = list(
docscrape_google.parse_google_argblock(lines))
for info in parsed_args:
_hack_for_info(info)
name = info['name'].replace('*', '')
name_to_parsed_docstr_info[name] = info
parsed_rets = list(
docscrape_google.parse_google_returns(real_func.__doc__))
ret_infos = []
for info in parsed_rets:
try:
got = fastparse.parse_type_string(
info['type'], 'Any', 0, 0)
ret_infos.append(got)
except Exception:
pass
# print('o = {!r}'.format(o))
# print('o.arguments = {!r}'.format(o.arguments))
args: List[str] = []
for i, arg_ in enumerate(o.arguments):
var = arg_.variable
kind = arg_.kind
name = var.name
annotated_type = (o.unanalyzed_type.arg_types[i] if isinstance(
o.unanalyzed_type, CallableType) else None)
if annotated_type is None:
if name in name_to_parsed_docstr_info:
name = name.replace('*', '')
doc_type_str = name_to_parsed_docstr_info[name].get(
'type', None)
if doc_type_str is not None:
doc_type_str = doc_type_str.split(', default')[0]
# annotated_type = doc_type_str
# import mypy.types as mypy_types
from mypy import fastparse
# globals_ = {**mypy_types.__dict__}
try:
# # got = mypy_types.deserialize_type(doc_type_str)
# got = eval(doc_type_str, globals_)
# got = mypy_types.get_proper_type(got)
# got = mypy_types.Iterable
got = fastparse.parse_type_string(
doc_type_str, 'Any', 0, 0)
except Exception as ex:
print('ex = {!r}'.format(ex))
print('Failed to parse doc_type_str = {!r}'.
format(doc_type_str))
else:
annotated_type = got
# print('PARSED: annotated_type = {!r}'.format(annotated_type))
# print('annotated_type = {!r}'.format(annotated_type))
# I think the name check is incorrect: there are libraries which
# name their 0th argument other than self/cls
is_self_arg = i == 0 and name == 'self'
is_cls_arg = i == 0 and name == 'cls'
annotation = ""
if annotated_type and not is_self_arg and not is_cls_arg:
# Luckily, an argument explicitly annotated with "Any" has
# type "UnboundType" and will not match.
if not isinstance(get_proper_type(annotated_type),
AnyType):
annotation = ": {}".format(
self.print_annotation(annotated_type))
# xdev change, where we try to port the defaults over to the stubs
# as well (otherwise they dont show up in the function help text)
XDEV_KEEP_SOME_DEFAULTS = True
if arg_.initializer:
if kind.is_named() and not any(
arg.startswith('*') for arg in args):
args.append('*')
if not annotation:
typename = self.get_str_type_of_node(
arg_.initializer, True, False)
if typename == '':
if XDEV_KEEP_SOME_DEFAULTS:
# TODO
annotation = '=...'
else:
annotation = '=...'
else:
annotation = ': {} = ...'.format(typename)
else:
if XDEV_KEEP_SOME_DEFAULTS:
import mypy
# arg_.initializer.is_special_form
if isinstance(
arg_.initializer,
(mypy.nodes.IntExpr, mypy.nodes.FloatExpr)):
annotation += '={!r}'.format(
arg_.initializer.value)
elif isinstance(arg_.initializer,
mypy.nodes.StrExpr):
annotation += '={!r}'.format(
arg_.initializer.value)
elif isinstance(arg_.initializer,
mypy.nodes.NameExpr):
annotation += '={}'.format(
arg_.initializer.name)
else:
# fallback, unhandled default
print(
f'todo: Unhandled arg_.initializer={type(arg_.initializer)}'
)
annotation += '=...'
else:
annotation += ' = ...'
arg = name + annotation
elif kind == ARG_STAR:
arg = '*%s%s' % (name, annotation)
elif kind == ARG_STAR2:
arg = '**%s%s' % (name, annotation)
else:
arg = name + annotation
args.append(arg)
retname = None
if o.name != '__init__' and isinstance(o.unanalyzed_type,
CallableType):
if isinstance(get_proper_type(o.unanalyzed_type.ret_type),
AnyType):
# Luckily, a return type explicitly annotated with "Any" has
# type "UnboundType" and will enter the else branch.
retname = None # implicit Any
else:
retname = self.print_annotation(o.unanalyzed_type.ret_type)
elif isinstance(o, FuncDef) and (o.is_abstract or o.name
in METHODS_WITH_RETURN_VALUE):
# Always assume abstract methods return Any unless explicitly annotated. Also
# some dunder methods should not have a None return type.
retname = None # implicit Any
elif has_yield_expression(o):
self.add_abc_import('Generator')
yield_name = 'None'
send_name = 'None'
return_name = 'None'
for expr, in_assignment in all_yield_expressions(o):
if expr.expr is not None and not self.is_none_expr(
expr.expr):
self.add_typing_import('Any')
yield_name = 'Any'
if in_assignment:
self.add_typing_import('Any')
send_name = 'Any'
if has_return_statement(o):
self.add_typing_import('Any')
return_name = 'Any'
generator_name = self.typing_name('Generator')
if return_parsed_docstr_info is not None:
print(
f'return_parsed_docstr_info={return_parsed_docstr_info}'
)
yield_name = return_parsed_docstr_info[1]
retname = f'{generator_name}[{yield_name}, {send_name}, {return_name}]'
# print('o.name = {}'.format(ub.repr2(o.name, nl=1)))
# print('retname = {!r}'.format(retname))
# print('retfield = {!r}'.format(retfield))
elif not has_return_statement(o) and not is_abstract:
retname = 'None'
if retname is None:
if return_parsed_docstr_info is not None:
retname = return_parsed_docstr_info[1]
retfield = ''
if retname is not None:
retfield = ' -> ' + retname
self.add(', '.join(args))
self.add("){}: ...\n".format(retfield))
self._state = FUNC
