def 取缓存目录(name):
"""
# The resource root directory is
# ~/AppData/Roaming on Windows,
# ~/.config on Linux and ~/Library/Application Support on Mac.
# The cache root directory is ~/AppData/Local on Windows,
# ~/.config on Linux and ~/Library/Caches on Mac.
"""
return ub.shrinkuser(ub.ensure_app_cache_dir(name))
def main(cmdline=True, **kw):
config = ListDeployedConfig(cmdline=cmdline, default=kw)
print('config = {}'.format(ub.repr2(dict(config), nl=1)))
runs_dpath = join(config['workdir'], 'fit/runs')
if not exists(runs_dpath):
print('Workdir does not seem to contain a runs dpath')
print('Checking for alternates? TODO')
raise NotImplementedError
workdirs = [config['workdir']]
for workdir in workdirs:
run_name = config['name']
if run_name is None:
named_run_dpath = join(runs_dpath, '*')
dpath_exists = exists(named_run_dpath)
print('dpath_exists = {!r}'.format(dpath_exists))
else:
named_run_dpath = join(runs_dpath, run_name)
dpath_exists = exists(named_run_dpath)
print('dpath_exists = {!r}'.format(dpath_exists))
deployed_fpaths = glob.glob(join(named_run_dpath, '*/*.zip'))
REMOVE_LINKS = 1
# TODO: do we always want to remove deploy.zip symlinks here?
if REMOVE_LINKS:
deployed_fpaths = [
fpath for fpath in deployed_fpaths if not os.path.islink(fpath)
]
sortby = 'last_modified'
if sortby:
deployed_fpaths = sorted(
deployed_fpaths,
key=lambda fpath: get_file_info(fpath)[sortby])
SHRINK = 1
if SHRINK:
# Make output text smaller, and more likely to work cross-system
deployed_fpaths = [
ub.shrinkuser(fpath, home='$HOME') for fpath in deployed_fpaths
]
print('deployed_fpaths = {}'.format(ub.repr2(deployed_fpaths, nl=1)))
def test_pathlib():
try:
import pathlib
base = pathlib.Path(ub.ensure_app_cache_dir('ubelt'))
dpath = base.joinpath('test_pathlib_mkdir')
# ensuredir
ub.delete(dpath)
assert not dpath.exists()
got = ub.ensuredir(dpath)
assert got.exists()
# shrinkuser
assert ub.shrinkuser(base) == '~/.cache/ubelt'
assert ub.augpath(base,
prefix='foo') == '/home/joncrall/.cache/fooubelt'
ub.expandpath(base)
except Exception:
import pytest
pytest.skip('pathlib is not installed')
def apply(registery, funcname, num_workers=0, **kwargs):
print(ub.color_text('--- APPLY {} ---'.format(funcname), 'white'))
print(' * num_workers = {!r}'.format(num_workers))
if num_workers == 0:
processed_repos = []
for repo in registery.repos:
print(ub.color_text('--- REPO = {} ---'.format(repo), 'blue'))
try:
getattr(repo, funcname)(**kwargs)
except DirtyRepoError:
print(
ub.color_text('Ignoring dirty repo={}'.format(repo),
'red'))
processed_repos.append(repo)
else:
from concurrent import futures
# with futures.ThreadPoolExecutor(max_workers=num_workers) as pool:
with futures.ProcessPoolExecutor(max_workers=num_workers) as pool:
tasks = []
for i, repo in enumerate(registery.repos):
future = pool.submit(worker, repo, funcname, kwargs)
future.repo = repo
tasks.append(future)
processed_repos = []
for future in futures.as_completed(tasks):
repo = future.repo
print(
ub.color_text('--- REPO = {} ---'.format(repo),
'blue'))
try:
repo = future.result()
except DirtyRepoError:
print(
ub.color_text(
'Ignoring dirty repo={}'.format(repo), 'red'))
else:
print(repo._getlogs())
processed_repos.append(repo)
print(
ub.color_text('--- FINISHED APPLY {} ---'.format(funcname),
'white'))
SHOW_CMDLOG = 1
if SHOW_CMDLOG:
print('LOGGED COMMANDS')
import os
ORIG_CWD = MY_CWD = os.getcwd()
for repo in processed_repos:
print('# --- For repo = {!r} --- '.format(repo))
for t in repo._logged_cmds:
cmd, cwd = t
if cwd is None:
cwd = os.get_cwd()
if cwd != MY_CWD:
print('cd ' + ub.shrinkuser(cwd))
MY_CWD = cwd
print(cmd)
print('cd ' + ub.shrinkuser(ORIG_CWD))
def cmd(command, shell=False, detach=False, verbose=0, tee=None, cwd=None,
env=None, tee_backend='auto', check=False, **kwargs):
"""
Executes a command in a subprocess.
The advantage of this wrapper around subprocess is that
(1) you control if the subprocess prints to stdout,
(2) the text written to stdout and stderr is returned for parsing,
(3) cross platform behavior that lets you specify the command as a string
or tuple regardless of whether or not shell=True.
(4) ability to detach, return the process object and allow the process to
run in the background (eventually we may return a Future object instead).
Args:
command (str or Sequence): bash-like command string or tuple of
executable and args
shell (bool, default=False): if True, process is run in shell.
detach (bool, default=False):
if True, process is detached and run in background.
verbose (int, default=0): verbosity mode. Can be 0, 1, 2, or 3.
tee (bool, optional): if True, simultaneously writes to stdout while
capturing output from the command. If not specified, defaults to
True if verbose > 0. If detach is True, then this argument is
ignored.
cwd (PathLike, optional): path to run command
env (str, optional): environment passed to Popen
tee_backend (str, optional): backend for tee output.
Valid choices are: "auto", "select" (POSIX only), and "thread".
check (bool, default=False): if True, check that the return code was
zero before returning, otherwise raise a CalledProcessError.
Does nothing if detach is True.
**kwargs: only used to support deprecated arguments
Returns:
dict: info - information about command status.
if detach is False ``info`` contains captured standard out,
standard error, and the return code
if detach is False ``info`` contains a reference to the process.
Notes:
Inputs can either be text or tuple based. On UNIX we ensure conversion
to text if shell=True, and to tuple if shell=False. On windows, the
input is always text based. See [3]_ for a potential cross-platform
shlex solution for windows.
CommandLine:
python -m ubelt.util_cmd cmd
python -c "import ubelt as ub; ub.cmd('ping localhost -c 2', verbose=2)"
References:
.. [1] https://stackoverflow.com/questions/11495783/redirect-subprocess-stderr-to-stdout
.. [2] https://stackoverflow.com/questions/7729336/how-can-i-print-and-display-subprocess-stdout-and-stderr-output-without-distorti
.. [3] https://stackoverflow.com/questions/33560364/python-windows-parsing-command-lines-with-shlex
Example:
>>> info = cmd(('echo', 'simple cmdline interface'), verbose=1)
simple cmdline interface
>>> assert info['ret'] == 0
>>> assert info['out'].strip() == 'simple cmdline interface'
>>> assert info['err'].strip() == ''
Example:
>>> info = cmd('echo str noshell', verbose=0)
>>> assert info['out'].strip() == 'str noshell'
Example:
>>> # windows echo will output extra single quotes
>>> info = cmd(('echo', 'tuple noshell'), verbose=0)
>>> assert info['out'].strip().strip("'") == 'tuple noshell'
Example:
>>> # Note this command is formatted to work on win32 and unix
>>> info = cmd('echo str&&echo shell', verbose=0, shell=True)
>>> assert info['out'].strip() == 'str' + chr(10) + 'shell'
Example:
>>> info = cmd(('echo', 'tuple shell'), verbose=0, shell=True)
>>> assert info['out'].strip().strip("'") == 'tuple shell'
Example:
>>> import pytest
>>> info = cmd('echo hi', check=True)
>>> import subprocess
>>> with pytest.raises(subprocess.CalledProcessError):
>>> cmd('exit 1', check=True, shell=True)
Example:
>>> import ubelt as ub
>>> from os.path import join, exists
>>> fpath1 = join(ub.get_app_cache_dir('ubelt'), 'cmdout1.txt')
>>> fpath2 = join(ub.get_app_cache_dir('ubelt'), 'cmdout2.txt')
>>> ub.delete(fpath1)
>>> ub.delete(fpath2)
>>> info1 = ub.cmd(('touch', fpath1), detach=True)
>>> info2 = ub.cmd('echo writing2 > ' + fpath2, shell=True, detach=True)
>>> while not exists(fpath1):
... pass
>>> while not exists(fpath2):
... pass
>>> assert ub.readfrom(fpath1) == ''
>>> assert ub.readfrom(fpath2).strip() == 'writing2'
>>> info1['proc'].wait()
>>> info2['proc'].wait()
"""
import subprocess
# TODO: stdout, stderr - experimental - custom file to pipe stdout/stderr to
if kwargs: # nocover
if 'verbout' in kwargs:
warnings.warn(
'`verbout` is deprecated and will be removed. '
'Use `tee` instead', DeprecationWarning)
tee = kwargs.pop('verbout')
if 'detatch' in kwargs:
warnings.warn(
'`detatch` is deprecated (misspelled) and will be removed. '
'Use `detach` instead', DeprecationWarning)
detach = kwargs.pop('detatch')
if kwargs:
raise ValueError('Unknown kwargs: {}'.format(list(kwargs.keys())))
# Determine if command is specified as text or a tuple
if isinstance(command, string_types):
command_text = command
command_tup = None
else:
import pipes
command_tup = command
command_text = ' '.join(list(map(pipes.quote, command_tup)))
if shell or sys.platform.startswith('win32'):
# When shell=True, args is sent to the shell (e.g. bin/sh) as text
args = command_text
else:
# When shell=False, args is a list of executable and arguments
if command_tup is None:
# parse this out of the string
# NOTE: perhaps use the solution from [3] here?
import shlex
command_tup = shlex.split(command_text)
# command_tup = shlex.split(command_text, posix=not WIN32)
args = command_tup
if tee is None:
tee = verbose > 0
if verbose > 1:
import os
import platform
import getpass
from ubelt import shrinkuser
if verbose > 2:
try:
print('┌─── START CMD ───')
except Exception: # nocover
print('+=== START CMD ===')
cwd_ = os.getcwd() if cwd is None else cwd
compname = platform.node()
username = getpass.getuser()
cwd_ = shrinkuser(cwd_)
ps1 = '[ubelt.cmd] {}@{}:{}$ '.format(username, compname, cwd_)
print(ps1 + command_text)
# Create a new process to execute the command
def make_proc():
# delay the creation of the process until we validate all args
proc = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=shell,
universal_newlines=True, cwd=cwd, env=env)
return proc
if detach:
info = {'proc': make_proc(), 'command': command_text}
if verbose > 0: # nocover
print('...detaching')
else:
if tee:
# We logging stdout and stderr, while simulaniously piping it to
# another stream.
stdout = sys.stdout
stderr = sys.stderr
proc = make_proc()
proc, logged_out, logged_err = _tee_output(proc, stdout, stderr,
backend=tee_backend)
try:
out = ''.join(logged_out)
except UnicodeDecodeError: # nocover
out = '\n'.join(_.decode('utf-8') for _ in logged_out)
try:
err = ''.join(logged_err)
except UnicodeDecodeError: # nocover
err = '\n'.join(_.decode('utf-8') for _ in logged_err)
(out_, err_) = proc.communicate()
else:
proc = make_proc()
(out, err) = proc.communicate()
# calling wait means that the process will terminate and it is safe to
# return a reference to the process object.
ret = proc.wait()
info = {
'out': out,
'err': err,
'ret': ret,
'proc': proc,
'cwd': cwd,
'command': command_text
}
if verbose > 2:
# https://en.wikipedia.org/wiki/Box-drawing_character
try:
print('└─── END CMD ───')
except Exception: # nocover
print('L___ END CMD ___')
if check:
if info['ret'] != 0:
if PY2 or PY34:
raise subprocess.CalledProcessError(
info['ret'], info['command'], info['out'])
else:
raise subprocess.CalledProcessError(
info['ret'], info['command'], info['out'], info['err'])
return info
def 下载文件缓存(url, 保存文件路径=None):
fpath = ub.grabdata(url, fpath=保存文件路径, verbose=0)
return ub.shrinkuser(fpath)
def 下载文件(url, 保存文件路径=None):
fpath = ub.download(url, fpath=保存文件路径, verbose=0)
return ub.shrinkuser(fpath)
def 路径_替换为用户路径(path, home='~'):
# 返回某个用户主目录的路径。
return ub.shrinkuser(path, home)
def test_compressuser_without_home():
username = basename(expanduser('~'))
not_the_user = '******' + username
ub.shrinkuser(not_the_user) == not_the_user