def main(): modnames = [ 'scriptconfig', 'kwarray', 'kwimage', 'kwplot', 'ndsampler', 'netharn', ] print('--- force module side effects --- ') for modname in modnames: ub.import_module_from_name(modname) print('--- begin module query ---') for modname in modnames: info = query_module_pypi_info(modname) print(ub.repr2(info)) if info['local_version'] > info['pypi_version']: print('--------') print("NEED TO PUBLISH {}".format(modname)) print('https://travis-ci.org/Erotemic/{}'.format(modname)) print('--------')
def main(): modnames = [ 'utool', 'vtool_ibeis', 'guitool_ibeis', 'plottool_ibeis', 'pyhesaff', 'pyflann_ibeis', 'ibeis', ] print('--- force module side effects --- ') for modname in modnames: ub.import_module_from_name(modname) print('--- begin module query ---') for modname in modnames: info = query_module_pypi_info(modname) print(ub.repr2(info)) if info['local_version'] > info['pypi_version']: print('--------') print("NEED TO PUBLISH {}".format(modname)) print('https://travis-ci.org/Erotemic/{}'.format(modname)) print('--------')
def gen_api_for_docs(): """ import sys, ubelt sys.path.append(ubelt.expandpath('~/code/ubelt/dev')) from gen_api_for_docs import * # NOQA """ from count_usage_freq import count_usage config, usage = count_usage(modname='kwimage') modname = config['modname'] gaurd = ('=' * 64 + ' ' + '=' * 16) print(gaurd) print('{:<64} {:>8}'.format(' Function name ', 'Usefulness')) print(gaurd) for key, value in usage.items(): print('{:<64} {:>16}'.format(':func:`' + modname + '.' + key + '`', value)) print(gaurd) import ubelt as ub module = ub.import_module_from_name(modname) attrnames = module.__all__ if hasattr(module, '__protected__'): # Hack for lazy imports for subattr in module.__protected__: submod = ub.import_module_from_name(modname + '.' + subattr) setattr(module, subattr, submod) attrnames += module.__protected__ for attrname in attrnames: member = getattr(module, attrname) submembers = getattr(member, '__all__', None) # if attrname.startswith('util_'): if not submembers: from mkinit.static_mkinit import _extract_attributes try: submembers = _extract_attributes(member.__file__) except AttributeError: pass if submembers: print('\n:mod:`{}.{}`'.format(modname, attrname)) print('-------------') for subname in submembers: if not subname.startswith('_'): print(':func:`{}.{}`'.format(modname, subname)) submembers = dir(member) print('config = ' + ub.repr2(config.asdict(), nl=1))
def code(self): if self._code is None: try: if self._expanded or self.type == 'Assign': # always use astunparse if we have expanded raise Exception # Attempt to dynamically extract the source code because it # keeps formatting better. module = ub.import_module_from_name(self.modname) obj = getattr(module, self.name) self._code = inspect.getsource(obj).strip('\n') except Exception: # Fallback on static sourcecode extraction # (NOTE: it should be possible to keep formatting with a bit of # work) self._code = unparse(self.node).strip('\n') return self._code
def query_module_pypi_info(modname, verbose=0): """ Determine the lastest version of a module on pypi and the current installed version. """ cmdinfo = ub.cmd('yolk -V {}'.format(modname), verbose=verbose, check=True) pypi_version = LooseVersion(cmdinfo['out'].strip().split(' ')[1]) try: module = ub.import_module_from_name(modname) except ImportError: local_version = None else: local_version = LooseVersion(module.__version__) info = { 'modname': modname, 'pypi_version': pypi_version, 'local_version': local_version, } return info
""" for modname in ut.ProgIter( AUTOLOAD_PLUGIN_MODNAMES, 'loading plugins', enabled=ut.VERYVERBOSE, adjust=False, freq=1, ): if isinstance(modname, tuple): flag, modname = modname if ut.get_argflag(flag): continue try: # ut.import_modname(modname) ub.import_module_from_name(modname) except ImportError: if 'wbia_cnn' in modname: import warnings warnings.warn('Unable to load plugin: {!r}'.format(modname)) else: raise # NOTE: new plugin code needs to be hacked in here currently # this is not a long term solution. THE Long term solution is to get these # working (which are partially integrated) # python -m wbia dev_autogen_explicit_imports # python -m wbia dev_autogen_explicit_injects # Ensure that all injectable modules are imported before constructing the
def _lookup_deprecated_attribute(key): import ubelt as ub # mapping from module name to the attributes that were moved there. refactored = { 'kwarray': [ 'ArrayAPI', 'DataFrameArray', 'DataFrameLight', 'FlatIndexer', 'LocLight', 'RunningStats', 'apply_grouping', 'arglexmax', 'argmaxima', 'argminima', 'atleast_nd', 'boolmask', 'ensure_rng', 'group_consecutive', 'group_consecutive_indices', 'group_indices', 'group_items', 'isect_flags', 'iter_reduce_ufunc', 'maxvalue_assignment', 'mincost_assignment', 'mindist_assignment', 'one_hot_embedding', 'one_hot_lookup', 'random_combinations', 'random_product', 'seed_global', 'setcover', 'shuffle', 'standard_normal', 'standard_normal32', 'standard_normal64', 'stats_dict', 'uniform', 'uniform32' ], 'kwimage': [ 'BASE_COLORS', 'Boxes', 'CSS4_COLORS', 'Color', 'Coords', 'Detections', 'Heatmap', 'Mask', 'MaskList', 'MultiPolygon', 'Points', 'PointsList', 'Polygon', 'PolygonList', 'Segmentation', 'SegmentationList', 'TABLEAU_COLORS', 'TORCH_GRID_SAMPLE_HAS_ALIGN', 'XKCD_COLORS', 'add_homog', 'atleast_3channels', 'available_nms_impls', 'convert_colorspace', 'daq_spatial_nms', 'decode_run_length', 'draw_boxes_on_image', 'draw_clf_on_image', 'draw_line_segments_on_image', 'draw_text_on_image', 'draw_vector_field', 'encode_run_length', 'ensure_alpha_channel', 'ensure_float01', 'ensure_uint255', 'fourier_mask', 'gaussian_patch', 'grab_test_image', 'grab_test_image_fpath', 'imread', 'imresize', 'imscale', 'imwrite', 'load_image_shape', 'make_channels_comparable', 'make_heatmask', 'make_orimask', 'make_vector_field', 'non_max_supression', 'normalize', 'num_channels', 'overlay_alpha_images', 'overlay_alpha_layers', 'radial_fourier_mask', 'remove_homog', 'rle_translate', 'smooth_prob', 'stack_images', 'stack_images_grid', 'subpixel_accum', 'subpixel_align', 'subpixel_getvalue', 'subpixel_maximum', 'subpixel_minimum', 'subpixel_set', 'subpixel_setvalue', 'subpixel_slice', 'subpixel_translate', 'warp_image', 'warp_points', 'warp_tensor', ], 'kwplot': [ 'BackendContext', 'Color', 'PlotNums', 'autompl', 'autoplt', 'distinct_colors', 'distinct_markers', 'draw_boxes', 'draw_boxes_on_image', 'draw_clf_on_image', 'draw_line_segments', 'draw_points', 'draw_text_on_image', 'ensure_fnum', 'figure', 'imshow', 'legend', 'make_conv_images', 'make_heatmask', 'make_legend_img', 'make_orimask', 'make_vector_field', 'multi_plot', 'next_fnum', 'plot_convolutional_features', 'plot_matrix', 'plot_surface3d', 'set_figtitle', 'set_mpl_backend', 'show_if_requested', ], 'ubelt': [ 'CacheStamp', ] } ERROR_ON_ACCESS = True for modname, attrs in refactored.items(): if key in attrs: text = ub.paragraph(''' The attribute `netharn.util.{key}` is deprecated. It was refactored and moved to `{modname}.{key}`. ''').format(key=key, modname=modname) if ERROR_ON_ACCESS: raise AttributeError(text) else: module = ub.import_module_from_name(modname) import warnings warnings.warn(text) return getattr(module, key) if key in ['SlidingIndexDataset', 'SlidingSlices']: raise AttributeError('Deprecated {}, but still available in ' 'netharn.util.util_slider_dep'.format(key)) raise AttributeError(key)
def test_import_modname_builtin(): module = ub.import_module_from_name('ast') import ast assert module is ast
def count_usage(cmdline=True): config = UsageConfig(cmdline=cmdline) import ubelt as ub import glob from os.path import join names = [ 'netharn', 'ndsampler', 'kwimage', 'kwplot', ] + config['extra_modnames'] all_fpaths = [] for name in names: if name: repo_fpath = ub.expandpath(join('~/code', name)) fpaths = glob.glob(join(repo_fpath, '**', '*.py'), recursive=True) for fpath in fpaths: all_fpaths.append((name, fpath)) print('names = {}'.format(ub.repr2(names))) import re import ubelt as ub modname = 'kwarray' module = ub.import_module_from_name(modname) package_name = module.__name__ package_allvar = module.__all__ pat = re.compile(r'\b' + package_name + r'\.(?P<attr>[a-zA-Z_][A-Za-z_0-9]*)\b') pkg_to_hist = ub.ddict(lambda: ub.ddict(int)) for name, fpath in ub.ProgIter(all_fpaths): # print('fpath = {!r}'.format(fpath)) text = ub.readfrom(fpath, verbose=0) # text = open(fpath, 'r').read() for match in pat.finditer(text): attr = match.groupdict()['attr'] if attr in package_allvar: pkg_to_hist[name][attr] += 1 hist_iter = iter(pkg_to_hist.values()) usage = next(hist_iter).copy() for other in hist_iter: for k, v in other.items(): usage[k] += v for attr in package_allvar: usage[attr] += 0 for name in pkg_to_hist.keys(): pkg_to_hist[name] = ub.odict( sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1]) usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1]) if config['print_packages']: print(ub.repr2(pkg_to_hist, nl=2)) if config['remove_zeros']: for k, v in list(usage.items()): if v == 0: usage.pop(k) # if config['hardcoded_ubelt_hack']: # for k in list(usage): # if k.startswith('util_'): # usage.pop(k) # if k.startswith('_util_'): # usage.pop(k) # # ub._util_deprecated # from ubelt import _util_deprecated # if k in dir(_util_deprecated): # usage.pop(k) print(ub.repr2(usage, nl=1)) return usage
def visit_func_def(self, o: FuncDef, is_abstract: bool = False, is_overload: bool = False) -> None: import ubelt as ub if (self.is_private_name(o.name, o.fullname) or self.is_not_in_all(o.name) or (self.is_recorded_name(o.name) and not is_overload)): self.clear_decorators() return if not self._indent and self._state not in ( EMPTY, FUNC) and not o.is_awaitable_coroutine: self.add('\n') if not self.is_top_level(): self_inits = find_self_initializers(o) for init, value in self_inits: if init in self.method_names: # Can't have both an attribute and a method/property with the same name. continue init_code = self.get_init(init, value) if init_code: self.add(init_code) # dump decorators, just before "def ..." for s in self._decorators: self.add(s) self.clear_decorators() self.add( "%s%sdef %s(" % (self._indent, 'async ' if o.is_coroutine else '', o.name)) self.record_name(o.name) # import ubelt as ub # if o.name == 'dzip': # import xdev # xdev.embed() def _hack_for_info(info): type_name = info['type'] if type_name is not None: results = hacked_typing_info(type_name) for typing_arg in results['typing_imports']: self.add_typing_import(typing_arg) for line in results['import_lines']: self.add_import_line(line) info['type'] = results['type_name'] name_to_parsed_docstr_info = {} return_parsed_docstr_info = None fullname = o.name if getattr(self, '_IN_CLASS', None) is not None: fullname = self._IN_CLASS + '.' + o.name curr = ub.import_module_from_name(self.module) # curr = sys.modules.get(self.module) # print('o.name = {!r}'.format(o.name)) # print('fullname = {!r}'.format(fullname)) for part in fullname.split('.'): # print('part = {!r}'.format(part)) # print('curr = {!r}'.format(curr)) curr = getattr(curr, part, None) # print('curr = {!r}'.format(curr)) real_func = curr # print('real_func = {!r}'.format(real_func)) # if o.name == 'dict_union': # import xdev # xdev.embed() if real_func is not None and real_func.__doc__ is not None: from mypy import fastparse from xdoctest.docstr import docscrape_google parsed_args = None # parsed_ret = None blocks = docscrape_google.split_google_docblocks( real_func.__doc__) # print('blocks = {}'.format(ub.repr2(blocks, nl=1))) for key, block in blocks: # print(f'key={key}') lines = block[0] if key == 'Returns': # print(f'lines={lines}') for retdict in docscrape_google.parse_google_retblock( lines): # print(f'retdict={retdict}') _hack_for_info(retdict) return_parsed_docstr_info = (key, retdict['type']) if return_parsed_docstr_info is None: print( 'Warning: return block for {} might be malformed' .format(real_func)) if key == 'Yields': for retdict in docscrape_google.parse_google_retblock( lines): _hack_for_info(retdict) return_parsed_docstr_info = (key, retdict['type']) if return_parsed_docstr_info is None: print( 'Warning: return block for {} might be malformed' .format(real_func)) if key == 'Args': # hack for *args lines = '\n'.join( [line.lstrip('*') for line in lines.split('\n')]) # print('lines = {!r}'.format(lines)) parsed_args = list( docscrape_google.parse_google_argblock(lines)) for info in parsed_args: _hack_for_info(info) name = info['name'].replace('*', '') name_to_parsed_docstr_info[name] = info parsed_rets = list( docscrape_google.parse_google_returns(real_func.__doc__)) ret_infos = [] for info in parsed_rets: try: got = fastparse.parse_type_string( info['type'], 'Any', 0, 0) ret_infos.append(got) except Exception: pass # print('o = {!r}'.format(o)) # print('o.arguments = {!r}'.format(o.arguments)) args: List[str] = [] for i, arg_ in enumerate(o.arguments): var = arg_.variable kind = arg_.kind name = var.name annotated_type = (o.unanalyzed_type.arg_types[i] if isinstance( o.unanalyzed_type, CallableType) else None) if annotated_type is None: if name in name_to_parsed_docstr_info: name = name.replace('*', '') doc_type_str = name_to_parsed_docstr_info[name].get( 'type', None) if doc_type_str is not None: doc_type_str = doc_type_str.split(', default')[0] # annotated_type = doc_type_str # import mypy.types as mypy_types from mypy import fastparse # globals_ = {**mypy_types.__dict__} try: # # got = mypy_types.deserialize_type(doc_type_str) # got = eval(doc_type_str, globals_) # got = mypy_types.get_proper_type(got) # got = mypy_types.Iterable got = fastparse.parse_type_string( doc_type_str, 'Any', 0, 0) except Exception as ex: print('ex = {!r}'.format(ex)) print('Failed to parse doc_type_str = {!r}'. format(doc_type_str)) else: annotated_type = got # print('PARSED: annotated_type = {!r}'.format(annotated_type)) # print('annotated_type = {!r}'.format(annotated_type)) # I think the name check is incorrect: there are libraries which # name their 0th argument other than self/cls is_self_arg = i == 0 and name == 'self' is_cls_arg = i == 0 and name == 'cls' annotation = "" if annotated_type and not is_self_arg and not is_cls_arg: # Luckily, an argument explicitly annotated with "Any" has # type "UnboundType" and will not match. if not isinstance(get_proper_type(annotated_type), AnyType): annotation = ": {}".format( self.print_annotation(annotated_type)) # xdev change, where we try to port the defaults over to the stubs # as well (otherwise they dont show up in the function help text) XDEV_KEEP_SOME_DEFAULTS = True if arg_.initializer: if kind.is_named() and not any( arg.startswith('*') for arg in args): args.append('*') if not annotation: typename = self.get_str_type_of_node( arg_.initializer, True, False) if typename == '': if XDEV_KEEP_SOME_DEFAULTS: # TODO annotation = '=...' else: annotation = '=...' else: annotation = ': {} = ...'.format(typename) else: if XDEV_KEEP_SOME_DEFAULTS: import mypy # arg_.initializer.is_special_form if isinstance( arg_.initializer, (mypy.nodes.IntExpr, mypy.nodes.FloatExpr)): annotation += '={!r}'.format( arg_.initializer.value) elif isinstance(arg_.initializer, mypy.nodes.StrExpr): annotation += '={!r}'.format( arg_.initializer.value) elif isinstance(arg_.initializer, mypy.nodes.NameExpr): annotation += '={}'.format( arg_.initializer.name) else: # fallback, unhandled default print( f'todo: Unhandled arg_.initializer={type(arg_.initializer)}' ) annotation += '=...' else: annotation += ' = ...' arg = name + annotation elif kind == ARG_STAR: arg = '*%s%s' % (name, annotation) elif kind == ARG_STAR2: arg = '**%s%s' % (name, annotation) else: arg = name + annotation args.append(arg) retname = None if o.name != '__init__' and isinstance(o.unanalyzed_type, CallableType): if isinstance(get_proper_type(o.unanalyzed_type.ret_type), AnyType): # Luckily, a return type explicitly annotated with "Any" has # type "UnboundType" and will enter the else branch. retname = None # implicit Any else: retname = self.print_annotation(o.unanalyzed_type.ret_type) elif isinstance(o, FuncDef) and (o.is_abstract or o.name in METHODS_WITH_RETURN_VALUE): # Always assume abstract methods return Any unless explicitly annotated. Also # some dunder methods should not have a None return type. retname = None # implicit Any elif has_yield_expression(o): self.add_abc_import('Generator') yield_name = 'None' send_name = 'None' return_name = 'None' for expr, in_assignment in all_yield_expressions(o): if expr.expr is not None and not self.is_none_expr( expr.expr): self.add_typing_import('Any') yield_name = 'Any' if in_assignment: self.add_typing_import('Any') send_name = 'Any' if has_return_statement(o): self.add_typing_import('Any') return_name = 'Any' generator_name = self.typing_name('Generator') if return_parsed_docstr_info is not None: print( f'return_parsed_docstr_info={return_parsed_docstr_info}' ) yield_name = return_parsed_docstr_info[1] retname = f'{generator_name}[{yield_name}, {send_name}, {return_name}]' # print('o.name = {}'.format(ub.repr2(o.name, nl=1))) # print('retname = {!r}'.format(retname)) # print('retfield = {!r}'.format(retfield)) elif not has_return_statement(o) and not is_abstract: retname = 'None' if retname is None: if return_parsed_docstr_info is not None: retname = return_parsed_docstr_info[1] retfield = '' if retname is not None: retfield = ' -> ' + retname self.add(', '.join(args)) self.add("){}: ...\n".format(retfield)) self._state = FUNC