""" from __future__ import absolute_import, division, print_function, unicode_literals import time import math import datetime from functools import partial from utool import util_logging from utool import util_inject from utool import util_arg from utool import util_time from utool import util_iter from utool import util_cplat from six.moves import range, zip import collections import six # NOQA print, rrr, profile = util_inject.inject2(__name__, '[progress]') default_timer = util_time.default_timer QUIET = util_arg.QUIET SILENT = util_arg.SILENT VALID_PROGRESS_TYPES = ['none', 'dots', 'fmtstr', 'simple'] AGGROFLUSH = util_arg.get_argflag('--aggroflush') PROGGRESS_BACKSPACE = not util_arg.get_argflag(('--screen', '--progress-backspace')) NO_PROGRESS = util_arg.get_argflag(('--no-progress', '--noprogress')) FORCE_ALL_PROGRESS = util_arg.get_argflag(('--force-all-progress',)) #('--screen' not in sys.argv and '--progress-backspace' not in sys.argv) DEBUG_FREQ_ADJUST = util_arg.get_argflag('--debug-adjust-freq')
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function try: import numpy as np HAVE_NUMPY = True except ImportError: HAVE_NUMPY = False # TODO remove numpy pass import operator from six.moves import zip from utool import util_iter from utool import util_alg from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[util_assert]') from utool import util_arg # NOQA def get_first_None_position(list_): for index, item in enumerate(list_): if item is None: return index return None def assert_raises(ex_type, func, *args, **kwargs): try: func(*args, **kwargs) except Exception as ex: assert isinstance(ex, ex_type), 'Should have raised an error' return True
from six.moves import builtins import inspect import textwrap import six import sys import functools import os from utool import util_print from utool import util_time from utool import util_iter from utool import util_dbg from utool import util_arg from utool import util_type from utool import util_inject from utool._internal import meta_util_six (print, rrr, profile) = util_inject.inject2(__name__, '[decor]') if util_type.HAVE_NUMPY: import numpy as np # Commandline to toggle certain convinience decorators SIG_PRESERVE = util_arg.get_argflag('--sigpreserve') #SIG_PRESERVE = not util_arg.SAFE or util_arg.get_argflag('--sigpreserve') ONEX_REPORT_INPUT = '--onex-report-input' in sys.argv #IGNORE_TRACEBACK = '--smalltb' in sys.argv or '--ignoretb' in sys.argv # FIXME: dupliated in _internal/py2_syntax_funcs IGNORE_TRACEBACK = not ('--nosmalltb' in sys.argv or '--noignoretb' in sys.argv) #if util_arg.STRICT: # IGNORE_TRACEBACK = False # do not ignore traceback when profiling
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function # , unicode_literals from six.moves import zip, filter, filterfalse, map, range # NOQA import six # NOQA #from os.path import split, dirname, join from os.path import dirname, join from utool import util_class # NOQA from utool import util_dev from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[util_project]') __GLOBAL_PROFILE__ = None def ensure_text(fname, text, repo_dpath='.', force=None, locals_={}, chmod=None): """ Args: fname (str): file name text (str): repo_dpath (str): directory path string(default = '.') force (bool): (default = False) locals_ (dict): (default = {}) Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA
You should opt to use a hash*27 function over a hash* function. TODO: the same hashing algorithm should be used everywhere Currently there is a mix of sha1, sha256, and sha512 in different places. """ from __future__ import absolute_import, division, print_function, unicode_literals import hashlib import copy import six import uuid import random from utool import util_inject (print, rrr, profile) = util_inject.inject2(__name__, '[hash]') # default length of hash codes HASH_LEN = 16 # HEX alphabet ALPHABET_16 = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' ] # A large base-54 alphabet (all chars are valid for filenames but not # pretty) ALPHABET_54 = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', ';', '=', '@', '[', ']', '^', '_', '`', '{',
# -*- coding: utf-8 -*- """ Utils for IPython/Jupyter Notebooks """ from __future__ import absolute_import, division, print_function, unicode_literals from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[ipynb]') def make_autogen_str(): r""" Returns: str: CommandLine: python -m utool.util_ipynb --exec-make_autogen_str --show Example: >>> # DISABLE_DOCTEST >>> from utool.util_ipynb import * # NOQA >>> import utool as ut >>> result = make_autogen_str() >>> print(result) """ import utool as ut import sys def get_regen_cmd(): # TODO: move to utool try: if len(sys.argv) > 0 and ut.checkpath(sys.argv[0]): # Check if running python command
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import sys import six import functools import re import types from utool import util_inject from utool._internal.meta_util_six import IntType, LongType, FloatType, BooleanType from utool._internal import meta_util_six #import warnings print, rrr, profile = util_inject.inject2(__name__, '[type]') __STR__ = meta_util_six.__STR__ if six.PY2: def type_str(type_): str_ = str(type_) str_ = str_.replace('<type \'', '').replace('\'>', '') str_ = str_.replace('<class \'', '').replace('\'>', '') return str_ VALID_STRING_TYPES = (str, unicode, basestring) else: def type_str(type_): return str(type_).replace('<class \'', '').replace('\'>', '') VALID_STRING_TYPES = (str,) # Very odd that I have to put in dtypes in two different ways.
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import sys import os from os.path import split, exists, join, dirname from utool import util_inject from utool._internal import meta_util_arg print, rrr, profile = util_inject.inject2(__name__, '[sysreq]') def in_virtual_env(): """ returns True if you are running inside a python virtual environment. (DOES NOT WORK IF IN IPYTHON AND USING A VIRTUALENV) sys.prefix gives the location of the virtualenv Notes: It seems IPython does not respect virtual environments properly. TODO: find a solution http://stackoverflow.com/questions/7335992/ipython-and-virtualenv-ignoring-site-packages References: http://stackoverflow.com/questions/1871549/python-determine-if-running-inside-virtualenv CommandLine: python -m utool.util_sysreq in_virtual_env Example: >>> # DISABLE_DOCTEST >>> from utool.util_sysreq import * # NOQA
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import os from utool import util_inject from utool import util_str print, rrr, profile = util_inject.inject2(__name__, "[resource]") try: # Resource does not exist in win32 import resource def time_in_usermode(): stime = resource.getrusage(resource.RUSAGE_SELF).ru_stime return stime def time_in_systemmode(): utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime return utime def peak_memory(): """Returns the resident set size (the portion of a process's memory that is held in RAM.) """ # MAXRSS is expressed in kilobytes. Convert to bytes # FIXME: MAXRSS is NOT expressed in kilobytes. use resource.getpagesize to Convert to bytes # References: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process # resource.getpagesize maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 return maxrss
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function # , unicode_literals from six.moves import zip, filter, filterfalse, map, range # NOQA import six # NOQA #from os.path import split, dirname, join from os.path import dirname, join from utool import util_class # NOQA from utool import util_dev from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[util_project]') __GLOBAL_PROFILE__ = None def ensure_text(fname, text, repo_dpath='.', force=None, locals_={}, chmod=None): """ Args: fname (str): file name text (str): repo_dpath (str): directory path string(default = '.') force (bool): (default = False) locals_ (dict): (default = {}) Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> result = setup_repo() >>> print(result) """
import re import textwrap try: import numpy as np except ImportError: pass from os.path import join, splitext, dirname # NOQA from utool import util_cplat from utool import util_path from utool import util_num from utool import util_dev from utool import util_io from utool import util_dbg from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[latex]') #def ensure_latex_environ(): # paths = os.environ['PATH'].split(os.pathsep) # mpl.rc('font',**{'family':'serif'}) # mpl.rc('text', usetex=True) # mpl.rc('text.latex',unicode=True) # mpl.rc('text.latex',preamble='\usepackage[utf8]{inputenc}') def find_ghostscript_exe(): import utool as ut if ut.WIN32: gs_exe = r'C:\Program Files (x86)\gs\gs9.16\bin\gswin32c.exe' else: gs_exe = 'gs'
ReloadingMetaclass KwargsWrapper """ from __future__ import absolute_import, division, print_function import sys import six import types import functools import collections import operator as op from collections import defaultdict from utool import util_inject from utool import util_set from utool import util_arg from utool._internal.meta_util_six import get_funcname, get_funcglobals print, rrr, profile = util_inject.inject2(__name__, '[class]', DEBUG=False) # Registers which classes have which attributes # FIXME: this might cause memory leaks # FIXME: this does cause weird reimport behavior __CLASSTYPE_ATTRIBUTES__ = defaultdict(util_set.oset) __CLASSTYPE_POSTINJECT_FUNCS__ = defaultdict(util_set.oset) __CLASSNAME_CLASSKEY_REGISTER__ = defaultdict(util_set.oset) #_rrr = rrr #def rrr(verbose=True): # """ keep registered functions through reloads ? """ # global __CLASSTYPE_ATTRIBUTES__ # global __CLASSTYPE_POSTINJECT_FUNCS__
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function import six import itertools try: import numpy as np except ImportError as ex: pass from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[util_numpy]') def tiled_range(range_, cols): return np.tile(np.arange(range_), (cols, 1)).T #np.tile(np.arange(num_qf).reshape(num_qf, 1), (1, k_vsmany)) def quantum_random(): """ returns a 32 bit unsigned integer quantum random number """ import quantumrandom data16 = quantumrandom.uint16(array_length=2) assert data16.flags['C_CONTIGUOUS'] data32 = data16.view(np.dtype('uint32'))[0] return data32 def ensure_rng(rng): if rng is None: rng = np.random if isinstance(rng, int): rng = np.random.RandomState(seed=rng)
from __future__ import absolute_import, division, print_function from os.path import dirname, split, join, splitext, exists, realpath, basename, commonprefix import six import sys import zipfile import tarfile import gzip import urllib # NOQA import functools import time from six.moves import urllib as _urllib from utool import util_path from utool import util_cplat from utool import util_arg from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[grabdata]') QUIET = util_arg.QUIET BadZipfile = zipfile.BadZipfile def archive_files(archive_fpath, fpath_list, small=True, allowZip64=False, overwrite=False, verbose=True, common_prefix=False): r""" Adds the files in ``fpath_list`` to an zip/tar archive. Args: archive_fpath (str): path to zipfile to create fpath_list (list): path of files to add to the zipfile small (bool): if True uses compression but the zipfile will take more
# -*- coding: utf-8 -*- """ SeeAlso: utool._internal.util_importer TODO: http://code.activestate.com/recipes/473888-lazy-module-imports/ https://pypi.python.org/pypi/zope.deferredimport/3.5.2 """ from __future__ import absolute_import, division, print_function, unicode_literals from utool import util_inject from utool import util_arg import sys print, rrr, profile = util_inject.inject2(__name__, '[import]') def possible_import_patterns(modname): """ does not support from x import * does not support from x import z, y Example: >>> import utool as ut >>> modname = 'package.submod.submod2.module' >>> result = ut.repr3(ut.possible_import_patterns(modname)) >>> print(result) [ 'import\\spackage.submod.submod2.module', 'from\\spackage\\.submod\\.submod2\\simportmodule', ] """ # common regexes
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import numpy as np from utool import util_inject (print, rrr, profile) = util_inject.inject2(__name__, '[depgraph_helpers]') def nx_topsort_nodes(graph, nodes): import utool as ut node_rank = ut.nx_topsort_rank(graph, nodes) node_idx = ut.rebase_labels(node_rank) sorted_nodes = ut.take(nodes, node_idx) return sorted_nodes def nx_topsort_rank(graph, nodes=None): """ graph = inputs.exi_graph.reverse() nodes = flat_node_order_ """ import networkx as nx import utool as ut topsort = list(nx.topological_sort(graph)) node_to_top_rank = ut.make_index_lookup(topsort) toprank = ut.dict_take(node_to_top_rank, nodes) return toprank def nx_common_descendants(graph, node1, node2): import networkx as nx descendants1 = nx.descendants(graph, node1)
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import os from utool import util_inject from utool import util_str print, rrr, profile = util_inject.inject2(__name__, '[resource]') try: # Resource does not exist in win32 import resource def time_in_usermode(): stime = resource.getrusage(resource.RUSAGE_SELF).ru_stime return stime def time_in_systemmode(): utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime return utime def peak_memory(): """Returns the resident set size (the portion of a process's memory that is held in RAM.) """ # MAXRSS is expressed in kilobytes. Convert to bytes # FIXME: MAXRSS is NOT expressed in kilobytes. use resource.getpagesize to Convert to bytes # References: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process #resource.getpagesize maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 return maxrss def get_resource_limits():
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals from os.path import join, splitext, basename from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[ubuntu]') def add_new_mimetype_association(ext, mime_name, exe_fpath=None, dry=True): """ TODO: move to external manager and generalize Args: ext (str): extension to associate mime_name (str): the name of the mime_name to create (defaults to ext) exe_fpath (str): executable location if this is for one specific file References: https://wiki.archlinux.org/index.php/Default_applications#Custom_file_associations Args: ext (str): extension to associate exe_fpath (str): executable location mime_name (str): the name of the mime_name to create (defaults to ext) CommandLine: python -m utool.util_ubuntu --exec-add_new_mimetype_association # Add ability to open ipython notebooks via double click python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=/usr/local/bin/ipynb python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=jupyter-notebook --force python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=sqlite --ext=.sqlite --exe-fpath=sqlitebrowser
import os import re import textwrap try: import numpy as np except ImportError: pass from os.path import join, splitext, dirname # NOQA from utool import util_cplat from utool import util_path from utool import util_num from utool import util_dev from utool import util_io from utool import util_dbg from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[latex]') #def ensure_latex_environ(): # paths = os.environ['PATH'].split(os.pathsep) # mpl.rc('font',**{'family':'serif'}) # mpl.rc('text', usetex=True) # mpl.rc('text.latex',unicode=True) # mpl.rc('text.latex',preamble='\usepackage[utf8]{inputenc}') def find_ghostscript_exe(): import utool as ut if ut.WIN32: gs_exe = r'C:\Program Files (x86)\gs\gs9.16\bin\gswin32c.exe' else: gs_exe = 'gs'
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals try: import numpy as np except ImportError as ex: pass from six.moves import zip, map import six from utool import util_type from utool import util_inject from utool import util_dev print, rrr, profile = util_inject.inject2(__name__, '[csv]') class CSV(util_dev.NiceRepr): def __init__(self, row_data): self.row_data = row_data self.header = row_data[0] self.header_tags = [[x] for x in self.header] self.short_header = None def __nice__(self): import utool as ut if self.short_header is None: header_str = ', '.join([ ut.truncate_str(h, maxlen=15, truncmsg='~//~') for h in self.header ]) else: header_str = ', '.join(self.short_header) return '(shape=%s: cols=%s)' % (
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function import re import os import six from collections import deque # NOQA from os.path import exists, dirname, join, expanduser, normpath from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[autogen]') class PythonStatement(object): """ Thin wrapper around a string representing executable python code """ def __init__(self, stmt): self.stmt = stmt def __repr__(self): return self.stmt def __str__(self): return self.stmt def dump_autogen_code(fpath, autogen_text, codetype='python', fullprint=None): """ Helper that write a file if -w is given on command line, otherwise it just prints it out. It has the opption of comparing a diff to the file. """ import utool as ut dowrite = ut.get_argflag(('-w', '--write')) show_diff = ut.get_argflag('--diff') num_context_lines = ut.get_argval('--diff', type_=int, default=None) show_diff = show_diff or num_context_lines is not None
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals import six import re import operator from utool import util_inject print, rrr, profile = util_inject.inject2(__name__, '[util_tags]') def modify_tags(tags_list, direct_map=None, regex_map=None, regex_aug=None, delete_unmapped=False, return_unmapped=False, return_map=False): import utool as ut tag_vocab = ut.unique(ut.flatten(tags_list)) alias_map = ut.odict() if regex_map is not None: alias_map.update(**ut.build_alias_map(regex_map, tag_vocab)) if direct_map is not None: alias_map.update(ut.odict(direct_map)) new_tags_list = tags_list new_tags_list = ut.alias_tags(new_tags_list, alias_map) if regex_aug is not None: alias_aug = ut.build_alias_map(regex_aug, tag_vocab) aug_tags_list = ut.alias_tags(new_tags_list, alias_aug) new_tags_list = [ut.unique(t1 + t2) for t1, t2 in zip(new_tags_list, aug_tags_list)] unmapped = list(set(tag_vocab) - set(alias_map.keys())) if delete_unmapped: new_tags_list = [ut.setdiff(tags, unmapped) for tags in new_tags_list]
TODO: the same hashing algorithm should be used everywhere Currently there is a mix of sha1, sha256, and sha512 in different places. """ from __future__ import absolute_import, division, print_function, unicode_literals import hashlib import copy import os import six import uuid import random import warnings from six.moves import zip, map from utool import util_inject from utool import util_path from utool import util_type (print, rrr, profile) = util_inject.inject2(__name__, '[hash]') if util_type.HAVE_NUMPY: import numpy as np # default length of hash codes HASH_LEN = 16 HASH_LEN2 = 32 # HEX alphabet ALPHABET_16 = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'] # A large base-54 alphabet (all chars are valid for filenames but not pretty) ALPHABET_54 = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
import six import sys import zipfile import tarfile import gzip import urllib # NOQA import functools import time import hashlib from six.moves import urllib as _urllib from utool import util_path from utool import util_cplat from utool import util_arg from utool import util_inject from utool import util_hash print, rrr, profile = util_inject.inject2(__name__) QUIET = util_arg.QUIET BadZipfile = zipfile.BadZipfile def archive_files(archive_fpath, fpath_list, small=True, allowZip64=False, overwrite=False, verbose=True, common_prefix=False): r""" Adds the files in `fpath_list` to an zip/tar archive. Args: archive_fpath (str): path to zipfile to create fpath_list (list): path of files to add to the zipfile small (bool): if True uses compression but the zipfile will take more