"""
from __future__ import absolute_import, division, print_function, unicode_literals
import time
import math
import datetime
from functools import partial
from utool import util_logging
from utool import util_inject
from utool import util_arg
from utool import util_time
from utool import util_iter
from utool import util_cplat
from six.moves import range, zip
import collections
import six # NOQA
print, rrr, profile = util_inject.inject2(__name__, '[progress]')
default_timer = util_time.default_timer
QUIET = util_arg.QUIET
SILENT = util_arg.SILENT
VALID_PROGRESS_TYPES = ['none', 'dots', 'fmtstr', 'simple']
AGGROFLUSH = util_arg.get_argflag('--aggroflush')
PROGGRESS_BACKSPACE = not util_arg.get_argflag(('--screen', '--progress-backspace'))
NO_PROGRESS = util_arg.get_argflag(('--no-progress', '--noprogress'))
FORCE_ALL_PROGRESS = util_arg.get_argflag(('--force-all-progress',))
#('--screen' not in sys.argv and '--progress-backspace' not in sys.argv)
DEBUG_FREQ_ADJUST = util_arg.get_argflag('--debug-adjust-freq')
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
try:
import numpy as np
HAVE_NUMPY = True
except ImportError:
HAVE_NUMPY = False
# TODO remove numpy
pass
import operator
from six.moves import zip
from utool import util_iter
from utool import util_alg
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_assert]')
from utool import util_arg # NOQA
def get_first_None_position(list_):
for index, item in enumerate(list_):
if item is None:
return index
return None
def assert_raises(ex_type, func, *args, **kwargs):
try:
func(*args, **kwargs)
except Exception as ex:
assert isinstance(ex, ex_type), 'Should have raised an error'
return True
from six.moves import builtins
import inspect
import textwrap
import six
import sys
import functools
import os
from utool import util_print
from utool import util_time
from utool import util_iter
from utool import util_dbg
from utool import util_arg
from utool import util_type
from utool import util_inject
from utool._internal import meta_util_six
(print, rrr, profile) = util_inject.inject2(__name__, '[decor]')
if util_type.HAVE_NUMPY:
import numpy as np
# Commandline to toggle certain convinience decorators
SIG_PRESERVE = util_arg.get_argflag('--sigpreserve')
#SIG_PRESERVE = not util_arg.SAFE or util_arg.get_argflag('--sigpreserve')
ONEX_REPORT_INPUT = '--onex-report-input' in sys.argv
#IGNORE_TRACEBACK = '--smalltb' in sys.argv or '--ignoretb' in sys.argv
# FIXME: dupliated in _internal/py2_syntax_funcs
IGNORE_TRACEBACK = not ('--nosmalltb' in sys.argv or '--noignoretb' in sys.argv)
#if util_arg.STRICT:
# IGNORE_TRACEBACK = False
# do not ignore traceback when profiling
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function # , unicode_literals
from six.moves import zip, filter, filterfalse, map, range # NOQA
import six # NOQA
#from os.path import split, dirname, join
from os.path import dirname, join
from utool import util_class # NOQA
from utool import util_dev
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_project]')
__GLOBAL_PROFILE__ = None
def ensure_text(fname,
text,
repo_dpath='.',
force=None,
locals_={},
chmod=None):
"""
Args:
fname (str): file name
text (str):
repo_dpath (str): directory path string(default = '.')
force (bool): (default = False)
locals_ (dict): (default = {})
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
You should opt to use a hash*27 function over a hash* function.
TODO: the same hashing algorithm should be used everywhere
Currently there is a mix of sha1, sha256, and sha512 in different places.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import hashlib
import copy
import six
import uuid
import random
from utool import util_inject
(print, rrr, profile) = util_inject.inject2(__name__, '[hash]')
# default length of hash codes
HASH_LEN = 16
# HEX alphabet
ALPHABET_16 = [
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
'f'
]
# A large base-54 alphabet (all chars are valid for filenames but not # pretty)
ALPHABET_54 = [
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
'u', 'v', 'w', 'x', 'y', 'z', ';', '=', '@', '[', ']', '^', '_', '`', '{',
# -*- coding: utf-8 -*-
"""
Utils for IPython/Jupyter Notebooks
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[ipynb]')
def make_autogen_str():
r"""
Returns:
str:
CommandLine:
python -m utool.util_ipynb --exec-make_autogen_str --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_ipynb import * # NOQA
>>> import utool as ut
>>> result = make_autogen_str()
>>> print(result)
"""
import utool as ut
import sys
def get_regen_cmd():
# TODO: move to utool
try:
if len(sys.argv) > 0 and ut.checkpath(sys.argv[0]):
# Check if running python command
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import six
import functools
import re
import types
from utool import util_inject
from utool._internal.meta_util_six import IntType, LongType, FloatType, BooleanType
from utool._internal import meta_util_six
#import warnings
print, rrr, profile = util_inject.inject2(__name__, '[type]')
__STR__ = meta_util_six.__STR__
if six.PY2:
def type_str(type_):
str_ = str(type_)
str_ = str_.replace('<type \'', '').replace('\'>', '')
str_ = str_.replace('<class \'', '').replace('\'>', '')
return str_
VALID_STRING_TYPES = (str, unicode, basestring)
else:
def type_str(type_):
return str(type_).replace('<class \'', '').replace('\'>', '')
VALID_STRING_TYPES = (str,)
# Very odd that I have to put in dtypes in two different ways.
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import os
from os.path import split, exists, join, dirname
from utool import util_inject
from utool._internal import meta_util_arg
print, rrr, profile = util_inject.inject2(__name__, '[sysreq]')
def in_virtual_env():
"""
returns True if you are running inside a python virtual environment.
(DOES NOT WORK IF IN IPYTHON AND USING A VIRTUALENV)
sys.prefix gives the location of the virtualenv
Notes:
It seems IPython does not respect virtual environments properly.
TODO: find a solution
http://stackoverflow.com/questions/7335992/ipython-and-virtualenv-ignoring-site-packages
References:
http://stackoverflow.com/questions/1871549/python-determine-if-running-inside-virtualenv
CommandLine:
python -m utool.util_sysreq in_virtual_env
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_sysreq import * # NOQA
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import sys
import os
from os.path import split, exists, join, dirname
from utool import util_inject
from utool._internal import meta_util_arg
print, rrr, profile = util_inject.inject2(__name__, '[sysreq]')
def in_virtual_env():
"""
returns True if you are running inside a python virtual environment.
(DOES NOT WORK IF IN IPYTHON AND USING A VIRTUALENV)
sys.prefix gives the location of the virtualenv
Notes:
It seems IPython does not respect virtual environments properly.
TODO: find a solution
http://stackoverflow.com/questions/7335992/ipython-and-virtualenv-ignoring-site-packages
References:
http://stackoverflow.com/questions/1871549/python-determine-if-running-inside-virtualenv
CommandLine:
python -m utool.util_sysreq in_virtual_env
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_sysreq import * # NOQA
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from utool import util_inject
from utool import util_str
print, rrr, profile = util_inject.inject2(__name__, "[resource]")
try:
# Resource does not exist in win32
import resource
def time_in_usermode():
stime = resource.getrusage(resource.RUSAGE_SELF).ru_stime
return stime
def time_in_systemmode():
utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime
return utime
def peak_memory():
"""Returns the resident set size (the portion of
a process's memory that is held in RAM.)
"""
# MAXRSS is expressed in kilobytes. Convert to bytes
# FIXME: MAXRSS is NOT expressed in kilobytes. use resource.getpagesize to Convert to bytes
# References: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
# resource.getpagesize
maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
return maxrss
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function # , unicode_literals
from six.moves import zip, filter, filterfalse, map, range # NOQA
import six # NOQA
#from os.path import split, dirname, join
from os.path import dirname, join
from utool import util_class # NOQA
from utool import util_dev
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_project]')
__GLOBAL_PROFILE__ = None
def ensure_text(fname, text, repo_dpath='.', force=None, locals_={}, chmod=None):
"""
Args:
fname (str): file name
text (str):
repo_dpath (str): directory path string(default = '.')
force (bool): (default = False)
locals_ (dict): (default = {})
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> result = setup_repo()
>>> print(result)
"""
import re
import textwrap
try:
import numpy as np
except ImportError:
pass
from os.path import join, splitext, dirname # NOQA
from utool import util_cplat
from utool import util_path
from utool import util_num
from utool import util_dev
from utool import util_io
from utool import util_dbg
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[latex]')
#def ensure_latex_environ():
# paths = os.environ['PATH'].split(os.pathsep)
# mpl.rc('font',**{'family':'serif'})
# mpl.rc('text', usetex=True)
# mpl.rc('text.latex',unicode=True)
# mpl.rc('text.latex',preamble='\usepackage[utf8]{inputenc}')
def find_ghostscript_exe():
import utool as ut
if ut.WIN32:
gs_exe = r'C:\Program Files (x86)\gs\gs9.16\bin\gswin32c.exe'
else:
gs_exe = 'gs'
ReloadingMetaclass
KwargsWrapper
"""
from __future__ import absolute_import, division, print_function
import sys
import six
import types
import functools
import collections
import operator as op
from collections import defaultdict
from utool import util_inject
from utool import util_set
from utool import util_arg
from utool._internal.meta_util_six import get_funcname, get_funcglobals
print, rrr, profile = util_inject.inject2(__name__, '[class]', DEBUG=False)
# Registers which classes have which attributes
# FIXME: this might cause memory leaks
# FIXME: this does cause weird reimport behavior
__CLASSTYPE_ATTRIBUTES__ = defaultdict(util_set.oset)
__CLASSTYPE_POSTINJECT_FUNCS__ = defaultdict(util_set.oset)
__CLASSNAME_CLASSKEY_REGISTER__ = defaultdict(util_set.oset)
#_rrr = rrr
#def rrr(verbose=True):
# """ keep registered functions through reloads ? """
# global __CLASSTYPE_ATTRIBUTES__
# global __CLASSTYPE_POSTINJECT_FUNCS__
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import six
import itertools
try:
import numpy as np
except ImportError as ex:
pass
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_numpy]')
def tiled_range(range_, cols):
return np.tile(np.arange(range_), (cols, 1)).T
#np.tile(np.arange(num_qf).reshape(num_qf, 1), (1, k_vsmany))
def quantum_random():
""" returns a 32 bit unsigned integer quantum random number """
import quantumrandom
data16 = quantumrandom.uint16(array_length=2)
assert data16.flags['C_CONTIGUOUS']
data32 = data16.view(np.dtype('uint32'))[0]
return data32
def ensure_rng(rng):
if rng is None:
rng = np.random
if isinstance(rng, int):
rng = np.random.RandomState(seed=rng)
from __future__ import absolute_import, division, print_function
from os.path import dirname, split, join, splitext, exists, realpath, basename, commonprefix
import six
import sys
import zipfile
import tarfile
import gzip
import urllib # NOQA
import functools
import time
from six.moves import urllib as _urllib
from utool import util_path
from utool import util_cplat
from utool import util_arg
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[grabdata]')
QUIET = util_arg.QUIET
BadZipfile = zipfile.BadZipfile
def archive_files(archive_fpath, fpath_list, small=True, allowZip64=False,
overwrite=False, verbose=True, common_prefix=False):
r"""
Adds the files in ``fpath_list`` to an zip/tar archive.
Args:
archive_fpath (str): path to zipfile to create
fpath_list (list): path of files to add to the zipfile
small (bool): if True uses compression but the zipfile will take more
# -*- coding: utf-8 -*-
"""
SeeAlso:
utool._internal.util_importer
TODO: http://code.activestate.com/recipes/473888-lazy-module-imports/
https://pypi.python.org/pypi/zope.deferredimport/3.5.2
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from utool import util_inject
from utool import util_arg
import sys
print, rrr, profile = util_inject.inject2(__name__, '[import]')
def possible_import_patterns(modname):
"""
does not support from x import *
does not support from x import z, y
Example:
>>> import utool as ut
>>> modname = 'package.submod.submod2.module'
>>> result = ut.repr3(ut.possible_import_patterns(modname))
>>> print(result)
[
'import\\spackage.submod.submod2.module',
'from\\spackage\\.submod\\.submod2\\simportmodule',
]
"""
# common regexes
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
from utool import util_inject
(print, rrr, profile) = util_inject.inject2(__name__, '[depgraph_helpers]')
def nx_topsort_nodes(graph, nodes):
import utool as ut
node_rank = ut.nx_topsort_rank(graph, nodes)
node_idx = ut.rebase_labels(node_rank)
sorted_nodes = ut.take(nodes, node_idx)
return sorted_nodes
def nx_topsort_rank(graph, nodes=None):
"""
graph = inputs.exi_graph.reverse()
nodes = flat_node_order_
"""
import networkx as nx
import utool as ut
topsort = list(nx.topological_sort(graph))
node_to_top_rank = ut.make_index_lookup(topsort)
toprank = ut.dict_take(node_to_top_rank, nodes)
return toprank
def nx_common_descendants(graph, node1, node2):
import networkx as nx
descendants1 = nx.descendants(graph, node1)
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from utool import util_inject
from utool import util_str
print, rrr, profile = util_inject.inject2(__name__, '[resource]')
try:
# Resource does not exist in win32
import resource
def time_in_usermode():
stime = resource.getrusage(resource.RUSAGE_SELF).ru_stime
return stime
def time_in_systemmode():
utime = resource.getrusage(resource.RUSAGE_SELF).ru_utime
return utime
def peak_memory():
"""Returns the resident set size (the portion of
a process's memory that is held in RAM.)
"""
# MAXRSS is expressed in kilobytes. Convert to bytes
# FIXME: MAXRSS is NOT expressed in kilobytes. use resource.getpagesize to Convert to bytes
# References: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
#resource.getpagesize
maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
return maxrss
def get_resource_limits():
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from os.path import join, splitext, basename
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[ubuntu]')
def add_new_mimetype_association(ext, mime_name, exe_fpath=None, dry=True):
"""
TODO: move to external manager and generalize
Args:
ext (str): extension to associate
mime_name (str): the name of the mime_name to create (defaults to ext)
exe_fpath (str): executable location if this is for one specific file
References:
https://wiki.archlinux.org/index.php/Default_applications#Custom_file_associations
Args:
ext (str): extension to associate
exe_fpath (str): executable location
mime_name (str): the name of the mime_name to create (defaults to ext)
CommandLine:
python -m utool.util_ubuntu --exec-add_new_mimetype_association
# Add ability to open ipython notebooks via double click
python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=/usr/local/bin/ipynb
python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=jupyter-notebook --force
python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=sqlite --ext=.sqlite --exe-fpath=sqlitebrowser
import os
import re
import textwrap
try:
import numpy as np
except ImportError:
pass
from os.path import join, splitext, dirname # NOQA
from utool import util_cplat
from utool import util_path
from utool import util_num
from utool import util_dev
from utool import util_io
from utool import util_dbg
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[latex]')
#def ensure_latex_environ():
# paths = os.environ['PATH'].split(os.pathsep)
# mpl.rc('font',**{'family':'serif'})
# mpl.rc('text', usetex=True)
# mpl.rc('text.latex',unicode=True)
# mpl.rc('text.latex',preamble='\usepackage[utf8]{inputenc}')
def find_ghostscript_exe():
import utool as ut
if ut.WIN32:
gs_exe = r'C:\Program Files (x86)\gs\gs9.16\bin\gswin32c.exe'
else:
gs_exe = 'gs'
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
try:
import numpy as np
except ImportError as ex:
pass
from six.moves import zip, map
import six
from utool import util_type
from utool import util_inject
from utool import util_dev
print, rrr, profile = util_inject.inject2(__name__, '[csv]')
class CSV(util_dev.NiceRepr):
def __init__(self, row_data):
self.row_data = row_data
self.header = row_data[0]
self.header_tags = [[x] for x in self.header]
self.short_header = None
def __nice__(self):
import utool as ut
if self.short_header is None:
header_str = ', '.join([
ut.truncate_str(h, maxlen=15, truncmsg='~//~')
for h in self.header
])
else:
header_str = ', '.join(self.short_header)
return '(shape=%s: cols=%s)' % (
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import re
import os
import six
from collections import deque # NOQA
from os.path import exists, dirname, join, expanduser, normpath
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[autogen]')
class PythonStatement(object):
""" Thin wrapper around a string representing executable python code """
def __init__(self, stmt):
self.stmt = stmt
def __repr__(self):
return self.stmt
def __str__(self):
return self.stmt
def dump_autogen_code(fpath, autogen_text, codetype='python', fullprint=None):
"""
Helper that write a file if -w is given on command line, otherwise
it just prints it out. It has the opption of comparing a diff to the file.
"""
import utool as ut
dowrite = ut.get_argflag(('-w', '--write'))
show_diff = ut.get_argflag('--diff')
num_context_lines = ut.get_argval('--diff', type_=int, default=None)
show_diff = show_diff or num_context_lines is not None
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import six
import re
import operator
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_tags]')
def modify_tags(tags_list, direct_map=None, regex_map=None, regex_aug=None,
delete_unmapped=False, return_unmapped=False,
return_map=False):
import utool as ut
tag_vocab = ut.unique(ut.flatten(tags_list))
alias_map = ut.odict()
if regex_map is not None:
alias_map.update(**ut.build_alias_map(regex_map, tag_vocab))
if direct_map is not None:
alias_map.update(ut.odict(direct_map))
new_tags_list = tags_list
new_tags_list = ut.alias_tags(new_tags_list, alias_map)
if regex_aug is not None:
alias_aug = ut.build_alias_map(regex_aug, tag_vocab)
aug_tags_list = ut.alias_tags(new_tags_list, alias_aug)
new_tags_list = [ut.unique(t1 + t2) for t1, t2 in zip(new_tags_list, aug_tags_list)]
unmapped = list(set(tag_vocab) - set(alias_map.keys()))
if delete_unmapped:
new_tags_list = [ut.setdiff(tags, unmapped) for tags in new_tags_list]
TODO: the same hashing algorithm should be used everywhere
Currently there is a mix of sha1, sha256, and sha512 in different places.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import hashlib
import copy
import os
import six
import uuid
import random
import warnings
from six.moves import zip, map
from utool import util_inject
from utool import util_path
from utool import util_type
(print, rrr, profile) = util_inject.inject2(__name__, '[hash]')
if util_type.HAVE_NUMPY:
import numpy as np
# default length of hash codes
HASH_LEN = 16
HASH_LEN2 = 32
# HEX alphabet
ALPHABET_16 = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'a', 'b', 'c', 'd', 'e', 'f']
# A large base-54 alphabet (all chars are valid for filenames but not pretty)
ALPHABET_54 = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
import six
import sys
import zipfile
import tarfile
import gzip
import urllib # NOQA
import functools
import time
import hashlib
from six.moves import urllib as _urllib
from utool import util_path
from utool import util_cplat
from utool import util_arg
from utool import util_inject
from utool import util_hash
print, rrr, profile = util_inject.inject2(__name__)
QUIET = util_arg.QUIET
BadZipfile = zipfile.BadZipfile
def archive_files(archive_fpath, fpath_list, small=True, allowZip64=False,
overwrite=False, verbose=True, common_prefix=False):
r"""
Adds the files in `fpath_list` to an zip/tar archive.
Args:
archive_fpath (str): path to zipfile to create
fpath_list (list): path of files to add to the zipfile
small (bool): if True uses compression but the zipfile will take more
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
try:
import numpy as np
HAVE_NUMPY = True
except ImportError:
HAVE_NUMPY = False
# TODO remove numpy
pass
import operator
from six.moves import zip
from utool import util_iter
from utool import util_alg
from utool import util_inject
print, rrr, profile = util_inject.inject2(__name__, '[util_assert]')
from utool import util_arg # NOQA
def get_first_None_position(list_):
for index, item in enumerate(list_):
if item is None:
return index
return None
def assert_raises(ex_type, func, *args, **kwargs):
try:
func(*args, **kwargs)
except Exception as ex:
assert isinstance(ex, ex_type), 'Should have raised an error'
return True
