def read_from(fpath, verbose=None, aslines=False, strict=True):
""" Reads text from a file
Args:
fpath (str): file path
aslines (bool): if True returns list of lines
verbose (bool): verbosity flag
Returns:
text from fpath
"""
if verbose or (verbose is None and __PRINT_READS__):
print('[util_io] * Reading text file: %r ' % util_path.tail(fpath))
try:
if not util_path.checkpath(fpath, verbose=verbose, n=3):
raise IOError('[io] * FILE DOES NOT EXIST!')
with open(fpath, 'r') as file_:
if aslines:
text = file_.readlines()
else:
text = file_.read()
return text
except IOError as ex:
from utool.util_dbg import printex
if verbose or strict:
printex(ex, ' * Error reading fpath=%r' %
util_path.tail(fpath), '[io]')
if strict:
raise
def latex_get_stats(lbl, data, mode=0):
stats_ = util_dev.get_stats(data)
if stats_.get('empty_list', False):
return '% NA: latex_get_stats, data=[]'
try:
max_ = stats_['max']
min_ = stats_['min']
mean = stats_['mean']
std = stats_['std']
shape = stats_['shape']
except KeyError as ex:
stat_keys = stats_.keys() # NOQA
printex(ex, key_list=['stat_keys', 'stats_', 'data'])
raise
#int_fmt = lambda num: util.num_fmt(int(num))
float_fmt = lambda num: util_num.num_fmt(float(num))
tup_fmt = lambda tup: str(tup)
fmttup = (float_fmt(min_), float_fmt(max_), float_fmt(mean), float_fmt(std), tup_fmt(shape))
lll = ' ' * len(lbl)
if mode == 0:
prefmtstr = r'''
{label} stats & min ; max = %s ; %s\\
{space} & mean; std = %s ; %s\\
{space} & shape = %s \\'''
if mode == 1:
prefmtstr = r'''
{label} stats & min = $%s$\\
{space} & max = $%s$\\
{space} & mean = $%s$\\
{space} & std = $%s$\\
{space} & shape = $%s$\\'''
fmtstr = prefmtstr.format(label=lbl, space=lll)
latex_str = textwrap.dedent(fmtstr % fmttup).strip('\n') + '\n'
return latex_str
def load(self):
""" Read pref dict stored on disk. Overwriting current values. """
if VERBOSE_PREF:
print('[pref.load()]')
#if not os.path.exists(self._intern.fpath):
# msg = '[pref] fpath=%r does not exist' % (self._intern.fpath)
# return msg
fpath = self.get_fpath()
try:
with open(fpath, 'rb') as f:
if VERBOSE_PREF:
print('load: %r' % fpath)
pref_dict = pickle.load(f)
except EOFError as ex1:
util_dbg.printex(ex1, 'did not load pref fpath=%r correctly' % fpath, iswarning=True)
#warnings.warn(msg)
raise
#return msg
except ImportError as ex2:
util_dbg.printex(ex2, 'did not load pref fpath=%r correctly' % fpath, iswarning=True)
#warnings.warn(msg)
raise
#return msg
if not util_type.is_dict(pref_dict):
raise Exception('Preference file is corrupted')
self.add_dict(pref_dict)
return True
def load(self):
""" Read pref dict stored on disk. Overwriting current values. """
if VERBOSE_PREF:
print('[pref.load()]')
#if not os.path.exists(self._intern.fpath):
# msg = '[pref] fpath=%r does not exist' % (self._intern.fpath)
# return msg
fpath = self.get_fpath()
try:
with open(fpath, 'r') as f:
if VERBOSE_PREF:
print('load: %r' % fpath)
pref_dict = pickle.load(f)
except EOFError as ex1:
util_dbg.printex(ex1,
'did not load pref fpath=%r correctly' % fpath,
iswarning=True)
#warnings.warn(msg)
raise
#return msg
except ImportError as ex2:
util_dbg.printex(ex2,
'did not load pref fpath=%r correctly' % fpath,
iswarning=True)
#warnings.warn(msg)
raise
#return msg
if not util_type.is_dict(pref_dict):
raise Exception('Preference file is corrupted')
self.add_dict(pref_dict)
return True
def latex_sanitize_command_name(_cmdname):
r"""
Args:
_cmdname (?):
Returns:
?: command_name
CommandLine:
python -m utool.util_latex --exec-latex_sanitize_command_name
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> _cmdname = '#foo bar.'
>>> command_name = latex_sanitize_command_name(_cmdname)
>>> result = ('command_name = %s' % (str(command_name),))
>>> print(result)
FooBar
"""
import re
import utool as ut
command_name = _cmdname
try:
def subroman(match):
import roman
try:
groupdict = match.groupdict()
num = int(groupdict['num'])
if num == 0:
return ''
return roman.toRoman(num)
except Exception as ex:
util_dbg.printex(ex, keys=['groupdict'])
raise
command_name = re.sub(ut.named_field('num', r'\d+'), subroman,
command_name)
except ImportError as ex:
if ut.SUPER_STRICT:
util_dbg.printex(ex)
raise
# remove numbers
command_name = re.sub(r'[\d' + re.escape('#()[]{}.') + ']', '',
command_name)
# Remove _ for camel case
#def to_camel_case(str_list):
# # hacky
# return ''.join([str_ if len(str_) < 1 else str_[0].upper() + str_[1:] for str_ in str_list])
#command_name = to_cammel_case(re.split('[_ ]', command_name)[::2])
str_list = re.split('[_ ]', command_name)
#command_name = to_cammel_case(str_list)
command_name = ut.to_camel_case('_'.join(str_list), mixed=True)
return command_name
def subroman(match):
import roman
try:
groupdict = match.groupdict()
num = int(groupdict['num'])
if num == 0:
return ''
return roman.toRoman(num)
except Exception as ex:
util_dbg.printex(ex, keys=['groupdict'])
raise
def subroman(match):
import roman
try:
groupdict = match.groupdict()
num = int(groupdict['num'])
if num == 0:
return ''
return roman.toRoman(num)
except Exception as ex:
util_dbg.printex(ex, keys=['groupdict'])
raise
def createQWidget(self):
# moving gui code away from utool
try:
#from utool._internal.PreferenceWidget import EditPrefWidget
from guitool.PreferenceWidget import EditPrefWidget
editpref_widget = EditPrefWidget(self)
editpref_widget.show()
return editpref_widget
except ImportError as ex:
util_dbg.printex(ex, 'Cannot create preference widget. Is guitool and PyQt 4/5 Installed')
raise
def read_from(fpath, verbose=None, aslines=False, strict=True, n=None, errors='replace'):
""" Reads text from a file. Automatically returns utf8.
Args:
fpath (str): file path
aslines (bool): if True returns list of lines
verbose (bool): verbosity flag
Returns:
str: text from fpath (this is unicode)
Ignore:
x = b'''/whaleshark_003_fors\xc3\xb8g.wmv" />\r\n'''
ut.writeto('foo.txt', x)
y = ut.readfrom('foo.txt')
y.encode('utf8') == x
"""
if n is None:
n = __READ_TAIL_N__
verbose = _rectify_verb_read(verbose)
if verbose:
print('[util_io] * Reading text file: %r ' % util_path.tail(fpath, n=n))
try:
if not util_path.checkpath(fpath, verbose=verbose, n=n):
raise IOError('[io] * FILE DOES NOT EXIST!')
#with open(fpath, 'r') as file_:
with open(fpath, 'rb') as file_:
if aslines:
#text = file_.readlines()
if six.PY2:
# python2 writes in bytes, so read as bytes then convert to
# utf8
text = [line.decode('utf8', errors=errors)
for line in file_.readlines()]
else:
text = [line.decode('utf8', errors=errors)
for line in file_.readlines()]
#text = file_.readlines()
else:
# text = file_.read()
if six.PY2:
text = file_.read().decode('utf8', errors=errors)
else:
#text = file_.read()
text = file_.read().decode('utf8', errors=errors)
return text
except IOError as ex:
from utool import util_dbg
if verbose or strict:
util_dbg.printex(ex, ' * Error reading fpath=%r' %
util_path.tail(fpath, n=n), '[io]')
if strict:
raise
def ensuredir(path_, verbose=VERYVERBOSE, info=False, mode=0o1777):
""" Ensures that directory will exist. creates new dir with sticky bits by default """
if not checkpath(path_, verbose=verbose, info=info):
if verbose:
print('[util_path] mkdir(%r)' % path_)
#os.makedirs(path_)
try:
os.makedirs(normpath(path_), mode=mode)
except OSError as ex:
util_dbg.printex(ex, 'check that the longest existing path is not a bad windows symlink.')
raise
return path_
def __enter__(self):
#self.shelf = get_global_shelf(self.appname)
try:
shelf_fpath = get_global_shelf_fpath(self.appname, ensure=True)
if VERBOSE:
print('[cache] open: ' + shelf_fpath)
self.shelf = shelve.open(shelf_fpath)
except Exception as ex:
from utool import util_dbg
util_dbg.printex(ex, 'Failed opening shelf_fpath',
key_list=['shelf_fpath'])
raise
return self.shelf
def latex_sanitize_command_name(_cmdname):
r"""
Args:
_cmdname (?):
Returns:
?: command_name
CommandLine:
python -m utool.util_latex --exec-latex_sanitize_command_name
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> _cmdname = '#foo bar.'
>>> command_name = latex_sanitize_command_name(_cmdname)
>>> result = ('command_name = %s' % (str(command_name),))
>>> print(result)
FooBar
"""
import re
import utool as ut
command_name = _cmdname
try:
def subroman(match):
import roman
try:
groupdict = match.groupdict()
num = int(groupdict['num'])
if num == 0:
return ''
return roman.toRoman(num)
except Exception as ex:
util_dbg.printex(ex, keys=['groupdict'])
raise
command_name = re.sub(ut.named_field('num', r'\d+'), subroman, command_name)
except ImportError as ex:
if ut.SUPER_STRICT:
util_dbg.printex(ex)
raise
# remove numbers
command_name = re.sub(r'[\d' + re.escape('#()[]{}.') + ']', '', command_name)
# Remove _ for camel case
#def to_camel_case(str_list):
# # hacky
# return ''.join([str_ if len(str_) < 1 else str_[0].upper() + str_[1:] for str_ in str_list])
#command_name = to_cammel_case(re.split('[_ ]', command_name)[::2])
str_list = re.split('[_ ]', command_name)
#command_name = to_cammel_case(str_list)
command_name = ut.to_camel_case('_'.join(str_list), mixed=True)
return command_name
def createQWidget(self):
# moving gui code away from utool
try:
#from utool._internal.PreferenceWidget import EditPrefWidget
from guitool.PreferenceWidget import EditPrefWidget
editpref_widget = EditPrefWidget(self)
editpref_widget.show()
return editpref_widget
except ImportError as ex:
util_dbg.printex(
ex,
'Cannot create preference widget. Is guitool and PyQt 4/5 Installed'
)
raise
def platform_path(path):
r"""
Returns platform specific path for pyinstaller usage
Args:
path (str):
Returns:
str: path2
CommandLine:
python -m utool.util_path --test-platform_path
Example:
>>> # ENABLE_DOCTEST
>>> # FIXME: find examples of the wird paths this fixes (mostly on win32 i think)
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> path = 'some/odd/../weird/path'
>>> path2 = platform_path(path)
>>> result = str(path2)
>>> if ut.WIN32:
... ut.assert_eq(path2, r'some\weird\path')
... else:
... ut.assert_eq(path2, r'some/weird/path')
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut # NOQA
>>> if ut.WIN32:
... path = 'C:/PROGRA~2'
... path2 = platform_path(path)
... assert path2 == u'..\\..\\..\\..\\Program Files (x86)'
"""
try:
if path == '':
raise ValueError('path cannot be the empty string')
# get path relative to cwd
path1 = truepath_relative(path)
if sys.platform.startswith('win32'):
path2 = fixwin32_shortname(path1)
else:
path2 = path1
except Exception as ex:
util_dbg.printex(ex, keys=['path', 'path1', 'path2'])
raise
return path2
def wrp_onexceptreport(*args, **kwargs):
try:
#import utool
#if utool.DEBUG:
# print('[IN EXCPRPT] args=%r' % (args,))
# print('[IN EXCPRPT] kwargs=%r' % (kwargs,))
return func(*args, **kwargs)
except Exception as ex:
from utool import util_str
arg_strs = ', '.join([repr(util_str.truncate_str(str(arg))) for arg in args])
kwarg_strs = ', '.join([util_str.truncate_str('%s=%r' % (key, val)) for key, val in six.iteritems(kwargs)])
msg = ('\nERROR: funcname=%r,\n * args=%s,\n * kwargs=%r\n' % (meta_util_six.get_funcname(func), arg_strs, kwarg_strs))
msg += ' * len(args) = %r\n' % len(args)
msg += ' * len(kwargs) = %r\n' % len(kwargs)
util_dbg.printex(ex, msg, pad_stdout=True)
raise
def assert_all_not_None(list_, list_name='some_list', key_list=[], verbose=not util_arg.QUIET,
veryverbose=False):
if util_arg.NO_ASSERTS:
return
try:
index = get_first_None_position(list_)
assert index is None, 'index=%r in %s is None' % (index, list_name)
if veryverbose:
print('PASSED: %s has no Nones' % (list_name))
except AssertionError as ex:
from utool import util_dbg
item = list_[index]
msg = (list_name + '[%d] = %r') % (index, item)
if verbose:
msg += '\n len(list_) = %r' % (len(list_))
util_dbg.printex(ex, msg, keys=key_list, N=1)
raise
def assert_in_setup_repo(setup_fpath, name=''):
""" pass in __file__ from setup.py """
setup_dir, setup_fname = split(setup_fpath)
cwd = os.getcwd()
#repo_dname = split(setup_dir)[1]
#print('cwd = %r' % (cwd))
#print('repo_dname = %r' % repo_dname)
#print('setup_dir = %r' % (setup_dir))
#print('setup_fname = %r' % (setup_fname))
try:
assert setup_fname == 'setup.py', 'name is not setup.py'
#assert name == '' or repo_dname == name,
('name=%r' % name)
assert cwd == setup_dir, 'cwd is not setup_dir'
assert exists(setup_dir), 'setup dir does not exist'
assert exists(join(setup_dir, 'setup.py')), 'setup.py does not exist'
except AssertionError as ex:
printex(ex, 'ERROR!: setup.py must be run from repository root')
raise
def _generate_parallel(func, args_list, ordered=True, chunksize=1,
prog=True, verbose=True, nTasks=None, freq=None):
"""
Parallel process generator
"""
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(1, nTasks // (__POOL__._processes ** 2))
if verbose:
prefix = '[util_parallel._generate_parallel]'
fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r'
print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize))
pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
result_generator = (
util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
if prog else raw_generator
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
for result in result_generator:
yield result
if __EAGER_JOIN__:
close_pool()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
close_pool()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(
func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)
def wrp_onexceptreport(*args, **kwargs):
try:
#import utool
#if utool.DEBUG:
# print('[IN EXCPRPT] args=%r' % (args,))
# print('[IN EXCPRPT] kwargs=%r' % (kwargs,))
return func(*args, **kwargs)
except Exception as ex:
from utool import util_str
print('ERROR occured! Reporting input to function')
if keys is not None:
from utool import util_inspect
from utool import util_list
from utool import util_dict
argspec = util_inspect.get_func_argspec(func)
in_kwargs_flags = [key in kwargs for key in keys]
kwarg_keys = util_list.compress(keys, in_kwargs_flags)
kwarg_vals = [kwargs.get(key) for key in kwarg_keys]
flags = util_list.not_list(in_kwargs_flags)
arg_keys = util_list.compress(keys, flags)
arg_idxs = [argspec.args.index(key) for key in arg_keys]
num_nodefault = len(argspec.args) - len(argspec.defaults)
default_vals = (([None] * (num_nodefault)) +
list(argspec.defaults))
args_ = list(args) + default_vals[len(args) + 1:]
arg_vals = util_list.take(args_, arg_idxs)
requested_dict = dict(util_list.flatten(
[zip(kwarg_keys, kwarg_vals), zip(arg_keys, arg_vals)]))
print('input dict = ' + util_str.repr4(
util_dict.dict_subset(requested_dict, keys)))
# (print out specific keys only)
pass
arg_strs = ', '.join([repr(util_str.truncate_str(str(arg)))
for arg in args])
kwarg_strs = ', '.join([
util_str.truncate_str('%s=%r' % (key, val))
for key, val in six.iteritems(kwargs)])
msg = ('\nERROR: funcname=%r,\n * args=%s,\n * kwargs=%r\n' % (
meta_util_six.get_funcname(func), arg_strs, kwarg_strs))
msg += ' * len(args) = %r\n' % len(args)
msg += ' * len(kwargs) = %r\n' % len(kwargs)
util_dbg.printex(ex, msg, pad_stdout=True)
raise
def assert_in_setup_repo(setup_fpath, name=''):
""" pass in __file__ from setup.py """
setup_dir, setup_fname = split(setup_fpath)
cwd = os.getcwd()
#repo_dname = split(setup_dir)[1]
#print('cwd = %r' % (cwd))
#print('repo_dname = %r' % repo_dname)
#print('setup_dir = %r' % (setup_dir))
#print('setup_fname = %r' % (setup_fname))
try:
assert setup_fname == 'setup.py', 'name is not setup.py'
#assert name == '' or repo_dname == name,
('name=%r' % name)
assert cwd == setup_dir, 'cwd is not setup_dir'
assert exists(setup_dir), 'setup dir does not exist'
assert exists(join(setup_dir, 'setup.py')), 'setup.py does not exist'
except AssertionError as ex:
printex(ex, 'ERROR!: setup.py must be run from repository root')
raise
def assert_all_not_None(list_,
list_name='some_list',
key_list=[],
verbose=not util_arg.QUIET,
veryverbose=False):
if util_arg.NO_ASSERTS:
return
try:
index = get_first_None_position(list_)
assert index is None, 'index=%r in %s is None' % (index, list_name)
if veryverbose:
print('PASSED: %s has no Nones' % (list_name))
except AssertionError as ex:
from utool import util_dbg
item = list_[index]
msg = (list_name + '[%d] = %r') % (index, item)
if verbose:
msg += '\n len(list_) = %r' % (len(list_))
util_dbg.printex(ex, msg, keys=key_list, N=1)
raise
def fixwin32_shortname(path1):
try:
#try:
# import win32file
# path2 = win32file.GetLongPathName(path1)
#except ImportError:
import ctypes
#import win32file
path1 = unicode(path1)
buflen = 260 # max size
buf = ctypes.create_unicode_buffer(buflen)
ctypes.windll.kernel32.GetLongPathNameW(path1, buf, buflen)
# If the path doesnt exist windows doesnt return anything
path2 = buf.value if len(buf.value) > 0 else path1
except Exception as ex:
print(ex)
util_dbg.printex(ex, 'cannot fix win32 shortcut', keys=['path1', 'path2'])
path2 = path1
#raise
return path2
def latex_get_stats(lbl, data, mode=0):
stats_ = util_dev.get_stats(data)
if stats_.get('empty_list', False):
return '% NA: latex_get_stats, data=[]'
try:
max_ = stats_['max']
min_ = stats_['min']
mean = stats_['mean']
std = stats_['std']
shape = stats_['shape']
except KeyError as ex:
stat_keys = stats_.keys() # NOQA
util_dbg.printex(ex, key_list=['stat_keys', 'stats_', 'data'])
raise
#int_fmt = lambda num: util.num_fmt(int(num))
def float_fmt(num):
return util_num.num_fmt(float(num))
def tup_fmt(tup):
return str(tup)
fmttup = (float_fmt(min_), float_fmt(max_), float_fmt(mean),
float_fmt(std), tup_fmt(shape))
lll = ' ' * len(lbl)
if mode == 0:
prefmtstr = r'''
{label} stats & min ; max = %s ; %s\\
{space} & mean; std = %s ; %s\\
{space} & shape = %s \\'''
if mode == 1:
prefmtstr = r'''
{label} stats & min = $%s$\\
{space} & max = $%s$\\
{space} & mean = $%s$\\
{space} & std = $%s$\\
{space} & shape = $%s$\\'''
fmtstr = prefmtstr.format(label=lbl, space=lll)
latex_str = textwrap.dedent(fmtstr % fmttup).strip('\n') + '\n'
return latex_str
def remove_fpaths(fpath_list, verbose=VERBOSE, quiet=QUIET, strict=False, print_caller=PRINT_CALLER, lbl='files'):
if print_caller:
print(util_dbg.get_caller_name(range(1, 4)) + ' called remove_fpaths')
nTotal = len(fpath_list)
_verbose = (not quiet and nTotal > 0) or VERYVERBOSE
if _verbose:
print('[util_path.remove_fpaths] try removing %d %s' % (nTotal, lbl))
nRemoved = 0
for fpath in fpath_list:
try:
os.remove(fpath) # Force refresh
nRemoved += 1
except OSError as ex:
if VERYVERBOSE:
print('WARNING: Could not remove fpath = %r' % (fpath,))
if strict:
util_dbg.printex(ex, 'Could not remove fpath = %r' % (fpath,), iswarning=False)
raise
pass
if _verbose:
print('[util_path.remove_fpaths] ... removed %d / %d %s' % (nRemoved, nTotal, lbl))
return nRemoved
def is_dicteq(dict1_, dict2_, almosteq_ok=True, verbose_err=True):
""" Checks to see if dicts are the same. Performs recursion. Handles numpy """
import utool
from utool import util_alg
from utool import util_dbg
assert len(dict1_) == len(dict2_), 'dicts are not of same length'
try:
for (key1, val1), (key2, val2) in zip(dict1_.items(), dict2_.items()):
assert key1 == key2, 'key mismatch'
assert type(val1) == type(val2), 'vals are not same type'
if HAVE_NUMPY and np.iterable(val1):
if almosteq_ok and utool.is_float(val1):
assert np.all(util_alg.almost_eq(val1, val2)), 'float vals are not within thresh'
else:
assert all([np.all(x1 == x2) for (x1, x2) in zip(val1, val2)]), 'np vals are different'
elif isinstance(val1, dict):
is_dicteq(val1, val2, almosteq_ok=almosteq_ok, verbose_err=verbose_err)
else:
assert val1 == val2, 'vals are different'
except AssertionError as ex:
if verbose_err:
util_dbg.printex(ex)
return False
return True
def _generate_parallel(func, args_list, ordered=True, chunksize=None,
prog=True, verbose=True, quiet=QUIET, nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if FUTURE_ON:
raise AssertionError('USE FUTURES')
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes ** 2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
verbose = not quiet
lbl = '(pargen) %s: ' % (get_funcname(func),)
result_generator = util_progress.ProgressIter(
raw_generator, nTotal=nTasks, lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False),
verbose=verbose
)
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
# DONT DO SERIAL FALLBACK IN GENERATOR CAN CAUSE ERRORS
raise
# print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
# if __SERIAL_FALLBACK__:
# print('Trying to handle error by falling back to serial')
# serial_generator = _generate_serial(
# func, args_list, prog=prog, verbose=verbose, nTasks=nTasks,
# **kwargs)
# for result in serial_generator:
# yield result
# else:
# raise
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_parallel(func,
args_list,
ordered=True,
chunksize=None,
prog=True,
verbose=True,
quiet=QUIET,
nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr %
(nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
lbl = '(pargen) %s: ' % (get_funcname(func), )
result_generator = util_progress.ProgressIter(
raw_generator,
nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False))
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(func,
args_list,
prog=prog,
verbose=verbose,
nTasks=nTasks,
**kwargs)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)
def make_score_tabular(
row_lbls, col_lbls, values, title=None, out_of=None, bold_best=False,
flip=False, bigger_is_better=True, multicol_lbls=None, FORCE_INT=False,
precision=None, SHORTEN_ROW_LBLS=False, col_align='l', col_sep='|',
multicol_sep='|', centerline=True, astable=False, table_position='',
AUTOFIX_LATEX=True, **kwargs):
r"""
makes a LaTeX tabular for displaying scores or errors
Args:
row_lbls (list of str):
col_lbls (list of str):
values (ndarray):
title (str): (default = None)
out_of (None): (default = None)
bold_best (bool): (default = True)
flip (bool): (default = False)
table_position (str) : eg '[h]'
Returns:
str: tabular_str
CommandLine:
python -m utool.util_latex --test-make_score_tabular:0 --show
python -m utool.util_latex --test-make_score_tabular:1 --show
python -m utool.util_latex --test-make_score_tabular:2 --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2], [3.2, 4]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
>>> multicol_lbls = [('spam', 1), ('eggs', 2)]
>>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
"""
import utool as ut
if flip:
bigger_is_better = not bigger_is_better
flip_repltups = [
('<=', '>'),
('>', '<='),
('\\leq', '\\gt'),
('\\geq', '\\lt'),
('score', 'error')
]
col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
if title is not None:
title = replace_all(title, flip_repltups)
if out_of is not None:
values = out_of - values
# Abbreviate based on common substrings
common_rowlbl = None
if SHORTEN_ROW_LBLS:
if isinstance(row_lbls, list):
row_lbl_list = row_lbls
else:
row_lbl_list = row_lbls.flatten().tolist()
# Split the rob labels into the alg components
#algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
longest = long_substr(row_lbl_list)
common_strs = []
while len(longest) > 10:
common_strs += [longest]
row_lbl_list = [row.replace(longest, '...') for row in row_lbl_list]
longest = long_substr(row_lbl_list)
common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
row_lbls = row_lbl_list
if len(row_lbl_list) == 1:
common_rowlbl = row_lbl_list[0]
row_lbls = ['0']
# Stack values into a tabular body
# TODO: need ability to specify datatypes
def ensurelist(row_values):
try:
return row_values.tolist()
except AttributeError:
return row_values
if False:
# Numpy formatting
def padvec(shape=(1, 1)):
pad = np.array([[' ' for c in range(shape[1])] for r in range(shape[0])])
return pad
col_lbls = ensure_rowvec(col_lbls)
row_lbls = ensure_colvec(row_lbls)
_0 = np.vstack([padvec(), row_lbls])
_1 = np.vstack([col_lbls, values])
body = np.hstack([_0, _1])
body = [[str_ for str_ in row] for row in body]
else:
assert len(row_lbls) == len(values)
body = [[' '] + col_lbls]
body += [[row_lbl] + ensurelist(row_values) for row_lbl, row_values in zip(row_lbls, values)]
#import utool as ut
# Fix things in each body cell
DO_PERCENT = True
try:
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if precision is not None:
# Hack
if ut.is_float(body[r][c]):
fmtstr = '%.' + str(precision) + 'f'
body[r][c] = fmtstr % (float(body[r][c]),)
# Force integer
if FORCE_INT:
body[r][c] = str(int(float(body[r][c])))
body[r][c] = str(body[r][c])
# Remove bad formatting;
if AUTOFIX_LATEX:
body[r][c] = escape_latex(body[r][c])
except Exception as ex:
import utool as ut
print('len(row_lbls) = %r' % (len(row_lbls),))
print('len(col_lbls) = %r' % (len(col_lbls),))
print('len(values) = %r' % (values,))
print('ut.depth_profile(values) = %r' % (ut.depth_profile(values),))
util_dbg.printex(ex, keys=['r', 'c'])
raise
# Bold the best values
if bold_best:
best_col_scores = values.max(0) if bigger_is_better else values.min(0)
rows_to_bold = [np.where(values[:, colx] == best_col_scores[colx])[0]
for colx in range(len(values.T))]
for colx, rowx_list in enumerate(rows_to_bold):
for rowx in rowx_list:
body[rowx + 1][colx + 1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'
# More fixing after the bold is in place
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if out_of is not None:
body[r][c] = body[r][c] + '/' + str(out_of)
if DO_PERCENT:
percent = ' = %.1f%%' % float(100 * values[r - 1, c - 1] / out_of)
body[r][c] += escape_latex(percent)
# Align columns for pretty printing
body = np.array(body)
ALIGN_BODY = True
if ALIGN_BODY:
new_body_cols = []
for col in body.T:
colstrs = list(map(str, ensurelist(col)))
collens = list(map(len, colstrs))
maxlen = max(collens)
newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
new_body_cols += [newcols]
body = np.array(new_body_cols).T
# Build Body (and row layout)
HLINE_SEP = True
rowvalsep = ''
colvalsep = ' & '
endl = '\\\\\n'
hline = r'\hline'
#extra_rowsep_pos_list = [1] # rows to insert an extra hline after
extra_rowsep_pos_list = [] # rows to insert an extra hline after
if HLINE_SEP:
rowvalsep = hline + '\n'
# rowstr list holds blocks of rows
rowstr_list = [colvalsep.join(row) + endl for row in body]
#rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
#rowstr_list = [(
# ('' if len(row) == 0 else row[0])
# if len(row) <= 1 else
# row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
# for row in body]
rowsep_list = [rowvalsep for row in rowstr_list[0:-1]] # should be len 1 less than rowstr_list
# Insert multicolumn names
if multicol_lbls is not None:
# TODO: label of the row labels
multicol_sep
multicols = [latex_multicolumn(multicol, size, 'c' + multicol_sep) for multicol, size in multicol_lbls]
multicol_str = latex_multirow('', 2) + colvalsep + colvalsep.join(multicols) + endl
ncols = sum([tup[1] for tup in multicol_lbls])
mcol_sep = '\\cline{2-%d}\n' % (ncols + 1,)
rowstr_list = [multicol_str] + rowstr_list
rowsep_list = [mcol_sep] + rowsep_list
#extra_rowsep_pos_list += [1]
# Insert title
if title is not None and not astable:
tex_title = latex_multicolumn(title, len(body[0])) + endl
rowstr_list = [tex_title] + rowstr_list
rowsep_list = [rowvalsep] + rowsep_list
#extra_rowsep_pos_list += [2]
# Apply an extra hline (for label)
#extra_rowsep_pos_list = []
for pos in sorted(extra_rowsep_pos_list)[::-1]:
rowstr_list.insert(pos, '')
rowsep_list.insert(pos, rowvalsep)
#tabular_body = rowvalsep.join(rowstr_list)
from six.moves import zip_longest
tabular_body = ''.join([row if sep is None else row + sep for row, sep in zip_longest(rowstr_list, rowsep_list)])
# Build Column Layout
col_align_list = [col_align] * len(body[0])
#extra_collayoutsep_pos_list = [1]
extra_collayoutsep_pos_list = []
for pos in sorted(extra_collayoutsep_pos_list)[::-1]:
col_align_list.insert(pos, '')
#col_layaout_sep_list = rowlblcol_sep # TODO
rowlblcol_sep = '|'
# Build build internal seprations between column alignments
# Defaults to just the normal col_sep
col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
# Adjust for the separations between row labels and the actual row data
if len(col_align_sep_list) > 0:
col_align_sep_list[0] = rowlblcol_sep
# Continue multicolumn sepratation
if multicol_lbls is not None:
multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
for offset in multicol_offsets:
if offset < len(col_align_sep_list):
col_align_sep_list[offset] = multicol_sep
from six.moves import zip_longest
_tmp = [ut.filter_Nones(tup) for tup in zip_longest(col_align_list, col_align_sep_list)]
col_layout = ''.join(ut.flatten(_tmp))
#if len(col_align_list) > 1:
# col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
#else:
# col_layout = col_sep.join(col_align_list)
tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
tabular_tail = r'\end{tabular}'
if centerline:
tabular_head = r'\centerline{' + '\n' + tabular_head
tabular_tail = tabular_tail + '}'
if astable:
#tabular_head = r'\begin{centering}' + '\n' + tabular_head
tabular_head = r'\centering' + '\n' + tabular_head
tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head
lblstr = latex_sanitize_command_name(kwargs.get('label', title))
caption = title
if AUTOFIX_LATEX:
caption = escape_latex(caption)
caption = '\n% ---\n' + caption + '\n% ---\n'
#tabular_head = r'\end{centering}' + '\n' + tabular_head
tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (lblstr, caption, lblstr)
tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
topsep = '\\hline\n' if True else '\\toprule\n'
botsep = '\\hline\n' if True else '\\bottomrule\n'
tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail
if common_rowlbl is not None:
#tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
pass
return tabular_str
def make_score_tabular(row_lbls,
col_lbls,
values,
title=None,
out_of=None,
bold_best=False,
flip=False,
bigger_is_better=True,
multicol_lbls=None,
FORCE_INT=False,
precision=None,
SHORTEN_ROW_LBLS=False,
col_align='l',
col_sep='|',
multicol_sep='|',
centerline=True,
astable=False,
table_position='',
AUTOFIX_LATEX=True,
**kwargs):
r"""
makes a LaTeX tabular for displaying scores or errors
Args:
row_lbls (list of str):
col_lbls (list of str):
values (ndarray):
title (str): (default = None)
out_of (None): (default = None)
bold_best (bool): (default = True)
flip (bool): (default = False)
table_position (str) : eg '[h]'
Returns:
str: tabular_str
CommandLine:
python -m utool.util_latex --test-make_score_tabular:0 --show
python -m utool.util_latex --test-make_score_tabular:1 --show
python -m utool.util_latex --test-make_score_tabular:2 --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2], [3.2, 4]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
>>> multicol_lbls = [('spam', 1), ('eggs', 2)]
>>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
"""
import utool as ut
if flip:
bigger_is_better = not bigger_is_better
flip_repltups = [('<=', '>'), ('>', '<='), ('\\leq', '\\gt'),
('\\geq', '\\lt'), ('score', 'error')]
col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
if title is not None:
title = replace_all(title, flip_repltups)
if out_of is not None:
values = out_of - values
# Abbreviate based on common substrings
common_rowlbl = None
if SHORTEN_ROW_LBLS:
if isinstance(row_lbls, list):
row_lbl_list = row_lbls
else:
row_lbl_list = row_lbls.flatten().tolist()
# Split the rob labels into the alg components
#algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
longest = long_substr(row_lbl_list)
common_strs = []
while len(longest) > 10:
common_strs += [longest]
row_lbl_list = [
row.replace(longest, '...') for row in row_lbl_list
]
longest = long_substr(row_lbl_list)
common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
row_lbls = row_lbl_list
if len(row_lbl_list) == 1:
common_rowlbl = row_lbl_list[0]
row_lbls = ['0']
# Stack values into a tabular body
# TODO: need ability to specify datatypes
def ensurelist(row_values):
try:
return row_values.tolist()
except AttributeError:
return row_values
if False:
# Numpy formatting
def padvec(shape=(1, 1)):
pad = np.array([[' ' for c in range(shape[1])]
for r in range(shape[0])])
return pad
col_lbls = ensure_rowvec(col_lbls)
row_lbls = ensure_colvec(row_lbls)
_0 = np.vstack([padvec(), row_lbls])
_1 = np.vstack([col_lbls, values])
body = np.hstack([_0, _1])
body = [[str_ for str_ in row] for row in body]
else:
assert len(row_lbls) == len(values)
body = [[' '] + col_lbls]
body += [[row_lbl] + ensurelist(row_values)
for row_lbl, row_values in zip(row_lbls, values)]
#import utool as ut
# Fix things in each body cell
DO_PERCENT = True
try:
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if precision is not None:
# Hack
if ut.is_float(body[r][c]):
fmtstr = '%.' + str(precision) + 'f'
body[r][c] = fmtstr % (float(body[r][c]), )
# Force integer
if FORCE_INT:
body[r][c] = str(int(float(body[r][c])))
body[r][c] = str(body[r][c])
# Remove bad formatting;
if AUTOFIX_LATEX:
body[r][c] = escape_latex(body[r][c])
except Exception as ex:
import utool as ut
print('len(row_lbls) = %r' % (len(row_lbls), ))
print('len(col_lbls) = %r' % (len(col_lbls), ))
print('len(values) = %r' % (values, ))
print('ut.depth_profile(values) = %r' % (ut.depth_profile(values), ))
util_dbg.printex(ex, keys=['r', 'c'])
raise
# Bold the best values
if bold_best:
best_col_scores = values.max(0) if bigger_is_better else values.min(0)
rows_to_bold = [
np.where(values[:, colx] == best_col_scores[colx])[0]
for colx in range(len(values.T))
]
for colx, rowx_list in enumerate(rows_to_bold):
for rowx in rowx_list:
body[rowx + 1][colx +
1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'
# More fixing after the bold is in place
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if out_of is not None:
body[r][c] = body[r][c] + '/' + str(out_of)
if DO_PERCENT:
percent = ' = %.1f%%' % float(
100 * values[r - 1, c - 1] / out_of)
body[r][c] += escape_latex(percent)
# Align columns for pretty printing
body = np.array(body)
ALIGN_BODY = True
if ALIGN_BODY:
new_body_cols = []
for col in body.T:
colstrs = list(map(str, ensurelist(col)))
collens = list(map(len, colstrs))
maxlen = max(collens)
newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
new_body_cols += [newcols]
body = np.array(new_body_cols).T
# Build Body (and row layout)
HLINE_SEP = True
rowvalsep = ''
colvalsep = ' & '
endl = '\\\\\n'
hline = r'\hline'
#extra_rowsep_pos_list = [1] # rows to insert an extra hline after
extra_rowsep_pos_list = [] # rows to insert an extra hline after
if HLINE_SEP:
rowvalsep = hline + '\n'
# rowstr list holds blocks of rows
rowstr_list = [colvalsep.join(row) + endl for row in body]
#rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
#rowstr_list = [(
# ('' if len(row) == 0 else row[0])
# if len(row) <= 1 else
# row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
# for row in body]
rowsep_list = [rowvalsep for row in rowstr_list[0:-1]
] # should be len 1 less than rowstr_list
# Insert multicolumn names
if multicol_lbls is not None:
# TODO: label of the row labels
multicol_sep
multicols = [
latex_multicolumn(multicol, size, 'c' + multicol_sep)
for multicol, size in multicol_lbls
]
multicol_str = latex_multirow(
'', 2) + colvalsep + colvalsep.join(multicols) + endl
ncols = sum([tup[1] for tup in multicol_lbls])
mcol_sep = '\\cline{2-%d}\n' % (ncols + 1, )
rowstr_list = [multicol_str] + rowstr_list
rowsep_list = [mcol_sep] + rowsep_list
#extra_rowsep_pos_list += [1]
# Insert title
if title is not None and not astable:
tex_title = latex_multicolumn(title, len(body[0])) + endl
rowstr_list = [tex_title] + rowstr_list
rowsep_list = [rowvalsep] + rowsep_list
#extra_rowsep_pos_list += [2]
# Apply an extra hline (for label)
#extra_rowsep_pos_list = []
for pos in sorted(extra_rowsep_pos_list)[::-1]:
rowstr_list.insert(pos, '')
rowsep_list.insert(pos, rowvalsep)
#tabular_body = rowvalsep.join(rowstr_list)
from six.moves import zip_longest
tabular_body = ''.join([
row if sep is None else row + sep
for row, sep in zip_longest(rowstr_list, rowsep_list)
])
# Build Column Layout
col_align_list = [col_align] * len(body[0])
#extra_collayoutsep_pos_list = [1]
extra_collayoutsep_pos_list = []
for pos in sorted(extra_collayoutsep_pos_list)[::-1]:
col_align_list.insert(pos, '')
#col_layaout_sep_list = rowlblcol_sep # TODO
rowlblcol_sep = '|'
# Build build internal seprations between column alignments
# Defaults to just the normal col_sep
col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
# Adjust for the separations between row labels and the actual row data
if len(col_align_sep_list) > 0:
col_align_sep_list[0] = rowlblcol_sep
# Continue multicolumn sepratation
if multicol_lbls is not None:
multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
for offset in multicol_offsets:
if offset < len(col_align_sep_list):
col_align_sep_list[offset] = multicol_sep
from six.moves import zip_longest
_tmp = [
ut.filter_Nones(tup)
for tup in zip_longest(col_align_list, col_align_sep_list)
]
col_layout = ''.join(ut.flatten(_tmp))
#if len(col_align_list) > 1:
# col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
#else:
# col_layout = col_sep.join(col_align_list)
tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
tabular_tail = r'\end{tabular}'
if centerline:
tabular_head = r'\centerline{' + '\n' + tabular_head
tabular_tail = tabular_tail + '}'
if astable:
#tabular_head = r'\begin{centering}' + '\n' + tabular_head
tabular_head = r'\centering' + '\n' + tabular_head
tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head
lblstr = latex_sanitize_command_name(kwargs.get('label', title))
caption = title
if AUTOFIX_LATEX:
caption = escape_latex(caption)
caption = '\n% ---\n' + caption + '\n% ---\n'
#tabular_head = r'\end{centering}' + '\n' + tabular_head
tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (
lblstr, caption, lblstr)
tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
topsep = '\\hline\n' if True else '\\toprule\n'
botsep = '\\hline\n' if True else '\\bottomrule\n'
tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail
if common_rowlbl is not None:
#tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
pass
return tabular_str
def write_to(fpath, to_write, aslines=False, verbose=None,
onlyifdiff=False, mode='w', n=None):
""" Writes text to a file. Automatically encodes text as utf8.
Args:
fpath (str): file path
to_write (str): text to write (must be unicode text)
aslines (bool): if True to_write is assumed to be a list of lines
verbose (bool): verbosity flag
onlyifdiff (bool): only writes if needed!
checks hash of to_write vs the hash of the contents of fpath
mode (unicode): (default = u'w')
n (int): (default = 2)
CommandLine:
python -m utool.util_io --exec-write_to --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_io import * # NOQA
>>> import utool as ut
>>> fpath = ut.unixjoin(ut.get_app_resource_dir('utool'), 'testwrite.txt')
>>> ut.delete(fpath)
>>> to_write = 'utf-8 symbols Δ, Й, ק, م, ๗, あ, 叶, 葉, and 말.'
>>> aslines = False
>>> verbose = True
>>> onlyifdiff = False
>>> mode = u'w'
>>> n = 2
>>> write_to(fpath, to_write, aslines, verbose, onlyifdiff, mode, n)
>>> read_ = ut.read_from(fpath)
>>> print('read_ = ' + read_)
>>> print('to_write = ' + to_write)
>>> assert read_ == to_write
"""
if onlyifdiff:
import utool as ut
if ut.hashstr(read_from(fpath)) == ut.hashstr(to_write):
print('[util_io] * no difference')
return
verbose = _rectify_verb_write(verbose)
if verbose:
# n = None if verbose > 1 else 2
# print('[util_io] * Writing to text file: %r ' % util_path.tail(fpath, n=n))
print('[util_io] * Writing to text file: {}'.format(fpath))
backup = False and exists(fpath)
if backup:
util_path.copy(fpath, fpath + '.backup')
if not isinstance(fpath, six.string_types):
# Assuming a file object with a name attribute
# Should just read from the file
fpath = fpath.name
with open(fpath, mode) as file_:
if aslines:
file_.writelines(to_write)
else:
# Ensure python2 writes in bytes
if six.PY2 and isinstance(to_write, unicode):
to_write = to_write.encode('utf8')
try:
file_.write(to_write)
except UnicodeEncodeError as ex:
start = max(ex.args[2] - 10, 0)
end = ex.args[3] + 10
context = to_write[start:end]
print(repr(context))
print(context)
from utool import util_dbg
util_dbg.printex(ex, keys=[(type, 'to_write')])
file_.close()
if backup:
# restore
util_path.copy(fpath + '.backup', fpath)
# import utool
# utool.embed()
raise
def read_from(fpath,
verbose=None,
aslines=False,
strict=True,
n=None,
errors='replace'):
""" Reads text from a file. Automatically returns utf8.
Args:
fpath (str): file path
aslines (bool): if True returns list of lines
verbose (bool): verbosity flag
Returns:
str: text from fpath (this is unicode)
Ignore:
x = b'''/whaleshark_003_fors\xc3\xb8g.wmv" />\r\n'''
ut.writeto('foo.txt', x)
y = ut.readfrom('foo.txt')
y.encode('utf8') == x
"""
if n is None:
n = __READ_TAIL_N__
verbose = _rectify_verb_read(verbose)
if verbose:
print('[util_io] * Reading text file: %r ' %
util_path.tail(fpath, n=n))
try:
if not util_path.checkpath(fpath, verbose=verbose, n=n):
raise IOError('[io] * FILE DOES NOT EXIST!')
#with open(fpath, 'r') as file_:
with open(fpath, 'rb') as file_:
if aslines:
#text = file_.readlines()
if six.PY2:
# python2 writes in bytes, so read as bytes then convert to utf8
text = [
line.decode('utf8', errors=errors)
for line in file_.readlines()
]
else:
text = [
line.decode('utf8', errors=errors)
for line in file_.readlines()
]
#text = file_.readlines()
else:
# text = file_.read()
if six.PY2:
text = file_.read().decode('utf8', errors=errors)
else:
#text = file_.read()
text = file_.read().decode('utf8', errors=errors)
return text
except IOError as ex:
from utool import util_dbg
if verbose or strict:
util_dbg.printex(
ex, ' * Error reading fpath=%r' % util_path.tail(fpath, n=n),
'[io]')
if strict:
raise
def presetup_commands(setup_fpath, kwargs):
if VERBOSE:
print('[setup] presetup_commands()')
name = kwargs.get('name', '')
# Parse args
project_dirs = kwargs.pop('project_dirs', None)
chmod_patterns = kwargs.pop('chmod_patterns', [])
clutter_dirs = kwargs.pop('clutter_dirs', None)
clutter_patterns = kwargs.pop('clutter_patterns', [])
build_command = kwargs.pop('build_command', NOOP)
# Augment patterns with builtin patterns
chmod_patterns += SETUP_PATTERNS.chmod_test if kwargs.pop('chmod_tests', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_pybuild if kwargs.pop('clean_pybuild', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_cyth if kwargs.pop('clean_pybuild', True) else []
setup_fpath = util_path.truepath(setup_fpath)
#
setup_dir = dirname(setup_fpath)
build_dir = join(setup_dir, 'build')
os.chdir(setup_dir) # change into setup directory
assert_in_setup_repo(setup_fpath, name)
# Augment clutter dirs
if clutter_dirs is None:
clutter_dirs = []
clutter_dirs += [
'build',
'dist',
name + '.egg-info',
'__pycache__'
]
if project_dirs is None:
project_dirs = util_path.ls_moduledirs(setup_dir)
# Execute pre-setup commands based on argv
#BEXT = 'bext' in sys.argv
#BUILD_EXT = 'build_ext' in sys.argv
#CYTH = 'cyth' in sys.argv
#if CYTH:
# translate_cyth()
for arg in iter(sys.argv[:]):
#print(arg)
# Clean clutter files
if arg in ['clean']:
clean(setup_dir, clutter_patterns, clutter_dirs)
#sys.exit(0)
if arg in ['build'] or (not exists(build_dir) and
'clean' not in sys.argv):
if VERBOSE:
print('[setup] Executing build command')
try:
build_command()
except Exception as ex:
printex(ex, 'Error calling buildcommand from cwd=%r\n' % os.getcwd())
raise
if arg in ['versions']:
print('+ ---')
print('Testing preqres package versions')
install_requires = kwargs.get('install_requires', [])
import pip
print('Checking install_requires = [\n%s\n]' % '\n'.join(install_requires))
pip.main(['show'] + [depline.split(' ')[0] for depline in install_requires])
print('L ___ Done Version Testing')
if arg in ['docs']:
# FIXME
autogen_sphinx_apidoc()
# Build optimized files
if arg in ['o', 'pyo']:
build_pyo(project_dirs)
# Cythonize files
#if arg in ['cython']:
# build_cython(cython_files)
# Chmod files
if arg in ['chmod']:
setup_chmod(setup_fpath, setup_dir, chmod_patterns)
try:
sys.argv.remove('docs')
sys.argv.remove('cyth')
#sys.argv.remove('--cyth-write')
except ValueError:
pass
try:
# SUPER HACK
# aliases bext to build_ext --inplace
sys.argv.remove('bext')
sys.argv.append('build_ext')
sys.argv.append('--inplace')
except ValueError:
pass
def presetup_commands(setup_fpath, kwargs):
if VERBOSE:
print('[setup] presetup_commands()')
name = kwargs.get('name', '')
# Parse args
project_dirs = kwargs.pop('project_dirs', None)
chmod_patterns = kwargs.pop('chmod_patterns', [])
clutter_dirs = kwargs.pop('clutter_dirs', None)
clutter_patterns = kwargs.pop('clutter_patterns', [])
build_command = kwargs.pop('build_command', NOOP)
# Augment patterns with builtin patterns
chmod_patterns += SETUP_PATTERNS.chmod_test if kwargs.pop(
'chmod_tests', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_pybuild if kwargs.pop(
'clean_pybuild', True) else []
clutter_patterns += SETUP_PATTERNS.clutter_cyth if kwargs.pop(
'clean_pybuild', True) else []
setup_fpath = util_path.truepath(setup_fpath)
#
setup_dir = dirname(setup_fpath)
build_dir = join(setup_dir, 'build')
os.chdir(setup_dir) # change into setup directory
assert_in_setup_repo(setup_fpath, name)
# Augment clutter dirs
if clutter_dirs is None:
clutter_dirs = []
clutter_dirs += ['build', 'dist', name + '.egg-info', '__pycache__']
if project_dirs is None:
project_dirs = util_path.ls_moduledirs(setup_dir)
# Execute pre-setup commands based on argv
#BEXT = 'bext' in sys.argv
#BUILD_EXT = 'build_ext' in sys.argv
#CYTH = 'cyth' in sys.argv
#if CYTH:
# translate_cyth()
for arg in iter(sys.argv[:]):
#print(arg)
# Clean clutter files
if arg in ['clean']:
clean(setup_dir, clutter_patterns, clutter_dirs)
#sys.exit(0)
if arg in ['build'] or (not exists(build_dir)
and 'clean' not in sys.argv):
if VERBOSE:
print('[setup] Executing build command')
try:
build_command()
except Exception as ex:
printex(
ex,
'Error calling buildcommand from cwd=%r\n' % os.getcwd())
raise
if arg in ['versions']:
print('+ ---')
print('Testing preqres package versions')
install_requires = kwargs.get('install_requires', [])
import pip
print('Checking install_requires = [\n%s\n]' %
'\n'.join(install_requires))
pip.main(['show'] +
[depline.split(' ')[0] for depline in install_requires])
print('L ___ Done Version Testing')
if arg in ['docs']:
# FIXME
autogen_sphinx_apidoc()
# Build optimized files
if arg in ['o', 'pyo']:
build_pyo(project_dirs)
# Cythonize files
#if arg in ['cython']:
# build_cython(cython_files)
# Chmod files
if arg in ['chmod']:
setup_chmod(setup_fpath, setup_dir, chmod_patterns)
try:
sys.argv.remove('docs')
sys.argv.remove('cyth')
#sys.argv.remove('--cyth-write')
except ValueError:
pass
try:
# SUPER HACK
# aliases bext to build_ext --inplace
sys.argv.remove('bext')
sys.argv.append('build_ext')
sys.argv.append('--inplace')
except ValueError:
pass
def write_to(fpath,
to_write,
aslines=False,
verbose=None,
onlyifdiff=False,
mode='w',
n=None):
""" Writes text to a file. Automatically encodes text as utf8.
Args:
fpath (str): file path
to_write (str): text to write (must be unicode text)
aslines (bool): if True to_write is assumed to be a list of lines
verbose (bool): verbosity flag
onlyifdiff (bool): only writes if needed!
checks hash of to_write vs the hash of the contents of fpath
mode (unicode): (default = u'w')
n (int): (default = 2)
CommandLine:
python -m utool.util_io --exec-write_to --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_io import * # NOQA
>>> import utool as ut
>>> fpath = ut.unixjoin(ut.get_app_resource_dir('utool'), 'testwrite.txt')
>>> ut.delete(fpath)
>>> to_write = 'utf-8 symbols Δ, Й, ק, م, ๗, あ, 叶, 葉, and 말.'
>>> aslines = False
>>> verbose = True
>>> onlyifdiff = False
>>> mode = u'w'
>>> n = 2
>>> write_to(fpath, to_write, aslines, verbose, onlyifdiff, mode, n)
>>> read_ = ut.read_from(fpath)
>>> print('read_ = ' + read_)
>>> print('to_write = ' + to_write)
>>> assert read_ == to_write
"""
if onlyifdiff:
import utool as ut
if ut.hashstr(read_from(fpath)) == ut.hashstr(to_write):
print('[util_io] * no difference')
return
verbose = _rectify_verb_write(verbose)
if verbose:
n = None if verbose > 1 else 2
print('[util_io] * Writing to text file: %r ' %
util_path.tail(fpath, n=n))
backup = False and exists(fpath)
if backup:
util_path.copy(fpath, fpath + '.backup')
if not isinstance(fpath, six.string_types):
# Assuming a file object with a name attribute
# Should just read from the file
fpath = fpath.name
with open(fpath, mode) as file_:
if aslines:
file_.writelines(to_write)
else:
# Ensure python2 writes in bytes
if six.PY2 and isinstance(to_write, unicode):
to_write = to_write.encode('utf8')
try:
file_.write(to_write)
except UnicodeEncodeError as ex:
start = max(ex.args[2] - 10, 0)
end = ex.args[3] + 10
context = to_write[start:end]
print(repr(context))
print(context)
from utool import util_dbg
util_dbg.printex(ex, keys=[(type, 'to_write')])
file_.close()
if backup:
# restore
util_path.copy(fpath + '.backup', fpath)
# import utool
# utool.embed()
raise
