def test_global_color_disable():
"""
CommandLine:
xdoctest -m /home/joncrall/code/ubelt/tests/test_color.py test_global_color_disable
"""
import ubelt as ub
text = 'text = "hi"'
has_color = ub.color_text(text, 'red') != ub.color_text(text, None)
text1a = ub.color_text(text, 'red')
text1b = ub.highlight_code(text)
if has_color:
assert text != text1a
assert text != text1b
# Force colors to be disabled
prev = ub.util_colors.NO_COLOR
try:
ub.util_colors.NO_COLOR = True
text2a = ub.color_text(text, 'red')
text2b = ub.highlight_code(text)
assert text == text2a
assert text == text2b
finally:
# Re-enable coloration
ub.util_colors.NO_COLOR = prev
text3a = ub.color_text(text, 'red')
text3b = ub.highlight_code(text)
if has_color:
assert text != text3a
assert text != text3b
def short_status(repo):
r"""
Example:
>>> repo = Repo(dpath=ub.truepath('.'))
>>> result = repo.short_status()
>>> print(result)
"""
import ubelt as ub
prefix = repo.dpath
with ChdirContext(repo.dpath, verbose=False):
info = ub.cmd('git status', verbose=False)
out = info['out']
# parse git status
is_clean_msg1 = 'Your branch is up-to-date with'
is_clean_msgs = [
'nothing to commit, working directory clean',
'nothing to commit, working tree clean',
]
msg2 = 'nothing added to commit but untracked files present'
needs_commit_msgs = [
'Changes to be committed',
'Changes not staged for commit',
'Your branch is ahead of',
]
suffix = ''
if is_clean_msg1 in out and any(msg in out for msg in is_clean_msgs):
suffix += ub.color_text('is clean', 'blue')
if msg2 in out:
suffix += ub.color_text('has untracked files', 'yellow')
if any(msg in out for msg in needs_commit_msgs):
suffix += ub.color_text('has changes', 'red')
print(prefix + ' ' + suffix)
def message(monitor, ansi=True):
"""
A status message with optional ANSI coloration
Args:
ansi (bool, default=True): if False disables ANSI coloration
Returns:
str: message for logging
Example:
>>> from netharn.monitor import *
>>> monitor = Monitor()
>>> print(monitor.message(ansi=False))
vloss is unevaluated
>>> monitor.update(0, {'loss': 1.0})
>>> print(monitor.message(ansi=False))
vloss: 1.0000 (n_bad=00, best=1.0000)
>>> monitor.update(0, {'loss': 2.0})
>>> print(monitor.message(ansi=False))
vloss: 1.4000 (n_bad=01, best=1.0000)
>>> monitor.update(0, {'loss': 0.1})
>>> print(monitor.message(ansi=False))
vloss: 0.8800 (n_bad=00, best=0.8800)
"""
if not monitor._epochs:
message = 'vloss is unevaluated'
if ansi:
message = ub.color_text(message, 'blue')
else:
prev_loss = monitor._smooth_metrics[-1]['loss']
best_loss = monitor._best_smooth_metrics['loss']
message = 'vloss: {:.4f} (n_bad={:02d}, best={:.4f})'.format(
prev_loss,
monitor._n_bad_epochs,
best_loss,
)
if monitor.patience is None:
patience = monitor.max_epoch
else:
patience = monitor.patience
if ansi:
if monitor._n_bad_epochs <= int(patience * .25):
message = ub.color_text(message, 'green')
elif monitor._n_bad_epochs >= int(patience * .75):
message = ub.color_text(message, 'red')
else:
message = ub.color_text(message, 'yellow')
return message
def main():
import sys
print(ub.color_text(' === TEST MULTIARG CLASSMETHOD ===', 'blue'))
print('sys.version_info = {!r}'.format(sys.version_info))
unbound_methods = {
# 'classmethod_self_args': MyType.classmethod_self_args,
# 'classmethod_args': MyType.classmethod_args,
# 'method_self_args': MyType.method_self_args,
'method_args': MyType.method_args,
'compat_method_args': MyType.compat_method_args,
}
for key, unbound in unbound_methods.items():
print('---')
print('key = {!r}'.format(key))
print('unbound = {!r}'.format(unbound))
try:
instance = MyType()
print('instance = {!r}'.format(instance))
bound = getattr(instance, key)
print('bound = {!r}'.format(bound))
print('Call as classmethod with 2 args')
unbound(instance, 42)
unbound(32, 42)
print('Call as bound method with 2 args')
bound(instance, 42)
bound(32, 42)
print('Call as classmethod with 1 args')
unbound(instance)
unbound(32)
print('Call as bound method with 1 args')
bound(instance)
bound(32)
print('Call as classmethod with 0 args')
unbound()
print('Call as bound method with 0 args')
bound()
except Exception as ex:
print(ub.color_text('XXX ex = {!r}'.format(ex), 'red'))
else:
print(ub.color_text('OOO worked', 'green'))
def _ldd_tree(fpath, depth=0):
prefix = '│ ' * (depth) + '├──'
if fpath in ldd_seen:
# if 'python' in fpath or 'py' in basename(fpath):
# print(ub.color_text(prefix + fpath + ' [SEEN]', 'red'))
# else:
# print(prefix + fpath + ' [SEEN]')
raise StopIteration()
else:
if print_tree:
if 'python' in fpath or 'py' in basename(fpath):
print(ub.color_text(prefix + fpath, 'red'))
else:
print(prefix + fpath)
if not exists(fpath):
yield fpath + ' (not found)'
# print('ERROR: does not exist fpath = {!r}'.format(fpath))
ldd_seen[fpath] = None
else:
yield fpath
# print('fpath = {!r}'.format(fpath))
if fpath not in ldd_seen:
ldd_seen[fpath] = list(ldd(fpath))
for child in ldd_seen[fpath]:
for _ in _ldd_tree(child, depth=depth + 1):
yield _
def message(early_stop):
if early_stop.prev_epoch is None:
return 'vloss is unevaluated'
# if early_stop.is_improved():
# message = 'vloss: {:.4f} (new_best)'.format(early_stop.best_loss)
message = 'vloss: {:.4f} (n_bad_epochs={:2d}, best={:.4f})'.format(
early_stop.prev_loss, early_stop.n_bad_epochs,
early_stop.best_loss,
)
if early_stop.n_bad_epochs <= int(early_stop.patience * .25):
message = ub.color_text(message, 'green')
elif early_stop.n_bad_epochs >= int(early_stop.patience * .75):
message = ub.color_text(message, 'red')
else:
message = ub.color_text(message, 'yellow')
return message
def predict(harn, have_true=True):
# Import the right version of caffe
print(ub.color_text('[segnet] begin prediction', 'blue'))
harn.prepare_test_model(force=False)
harn.test.make_dumpsafe_names()
net = harn.make_net()
assert harn.test_batch_size == 1
# have_true = bool(harn.test.gt_paths)
if not have_true:
def load_batch_data(bx):
"""
bx = 0
"""
offset = bx * harn.test_batch_size
blob_data = net.blobs['data'].data
for jx in range(harn.test_batch_size):
# push data into the network
ix = offset + jx
im_hwc = util.imread(harn.test.im_paths[ix])
im_hwc = im_hwc[:, :, ::-1]
im_chw = np.transpose(im_hwc, (2, 0, 1)).astype(np.float32)
blob_data[jx, :, :, :] = im_chw
n_iter = int(harn.test.n_input / harn.test_batch_size)
for bx in ub.ProgIter(range(n_iter), label='forward batch', freq=1):
if not have_true:
load_batch_data(bx)
net.forward()
blobs = net.blobs
harn.dump_predictions(blobs, bx, have_true=have_true)
def test_unable_to_find_color():
import ubelt as ub
import pytest
if ub.util_colors.NO_COLOR:
pytest.skip()
with pytest.warns(UserWarning):
text = ub.color_text('text', 'wizbang')
assert text == 'text', 'bad colors should pass the text back'
def message(monitor):
if not monitor.epochs:
return ub.color_text('vloss is unevaluated', 'blue')
# if monitor.is_improved():
# message = 'vloss: {:.4f} (new_best)'.format(monitor.best_loss)
prev_loss = monitor.smooth_metrics[-1]['loss']
best_loss = monitor.best_smooth_metrics['loss']
message = 'vloss: {:.4f} (n_bad_epochs={:2d}, best={:.4f})'.format(
prev_loss, monitor.n_bad_epochs, best_loss,
)
if monitor.n_bad_epochs <= int(monitor.patience * .25):
message = ub.color_text(message, 'green')
elif monitor.n_bad_epochs >= int(monitor.patience * .75):
message = ub.color_text(message, 'red')
else:
message = ub.color_text(message, 'yellow')
return message
def bench_isinstance_vs_attr():
instances = {
'base1': Base1(),
'base2': Base2(),
'derived2': Derived2(),
}
import ubelt as ub
ti = ub.Timerit(100000, bestof=500, verbose=1, unit='us')
# Do this twice, but keep the second measure
data = ub.AutoDict()
for selfname, self in instances.items():
print(ub.color_text('--- SELF = {} ---'.format(selfname), 'blue'))
subdata = data[selfname] = {}
for timer in ti.reset('isinstance(self, Base1)'):
with timer:
isinstance(self, Base1)
subdata[ti.label] = ti.min()
for timer in ti.reset('isinstance(self, Base2)'):
with timer:
isinstance(self, Base2)
subdata[ti.label] = ti.min()
for timer in ti.reset('isinstance(self, Derived2)'):
with timer:
isinstance(self, Derived2)
subdata[ti.label] = ti.min()
for timer in ti.reset('getattr(self, "class_attr1", False)'):
with timer:
getattr(self, 'class_attr1', False)
subdata[ti.label] = ti.min()
for timer in ti.reset('getattr(self, "attr1", False)'):
with timer:
getattr(self, 'attr1', False)
subdata[ti.label] = ti.min()
try:
import pandas as pd
df = pd.DataFrame(data) * 1e9
try:
from kwil.util.util_pandas import _to_string_monkey
print(_to_string_monkey(df, key='minima'))
except Exception:
print(df)
except ImportError:
print('no pandas')
print(ub.repr2(data, nl=2, precision=4))
def humanize_dollars(dollars, named=False, colored=False):
if named:
text = '${}'.format(named_large_number(dollars))
# return '${:5.02g}'.format(float(dollars))
else:
text = '${:5.02g}'.format(float(dollars))
if colored:
if dollars > 1e16:
# Blue is VERY safe
text = ub.color_text(text, 'blue')
elif dollars > 1e6:
# Green is safe
text = ub.color_text(text, 'green')
elif dollars > 1e3:
# Yellow is short-term safe
text = ub.color_text(text, 'yellow')
else:
# Red is not safe
text = ub.color_text(text, 'red')
return text
def cprint(text, color=None):
""" provides some color to terminal output
Args:
text (str):
color (str):
Ignore:
assert color in ['', 'yellow', 'blink', 'lightgray', 'underline',
'darkyellow', 'blue', 'darkblue', 'faint', 'fuchsia', 'black', 'white',
'red', 'brown', 'turquoise', 'bold', 'darkred', 'darkgreen', 'reset',
'standout', 'darkteal', 'darkgray', 'overline', 'purple', 'green', 'teal',
'fuscia']
Example0:
>>> import pygments.console
>>> msg_list = list(pygments.console.codes.keys())
>>> color_list = list(pygments.console.codes.keys())
>>> [cprint(text, color) for text, color in zip(msg_list, color_list)]
Example1:
>>> import pygments.console
>>> print('line1')
>>> cprint('line2', 'red')
>>> cprint('line3', 'blue')
>>> cprint('line4', 'fuchsia')
>>> cprint('line5', 'reset')
>>> cprint('line5', 'fuchsia')
>>> print('line6')
"""
if False and ub.WIN32:
# Ignore colors on windows. Seems to cause a recursion error
print(text)
else:
try:
if color is None:
print(ub.color_text(text, 'blue'))
else:
print(ub.color_text(text, color))
except RecursionError:
print(text)
def highlight_regex(str_, pat, reflags=0, color='red'):
"""
FIXME Use pygments instead
"""
import re
matches = list(re.finditer(pat, str_, flags=reflags))
colored = str_
for match in reversed(matches):
start = match.start()
end = match.end()
colored_part = ub.color_text(colored[start:end], color)
colored = colored[:start] + colored_part + colored[end:]
return colored
def humanize_seconds(seconds, colored=True, precision=4, named=False):
minutes = seconds / 60.
hours = minutes / 60.
days = hours / 24.
years = days / 365.
if minutes < 1:
raw = (seconds, 'seconds')
elif hours < 1:
raw = (minutes, 'minutes')
elif days < 1:
raw = (hours, 'hours')
elif years < 1:
raw = (days, 'days')
else:
raw = (years, 'years')
count, unit = raw
count_ = round(float(count), 4)
if named:
ret = '{} {}'.format(named_large_number(count), unit)
else:
ret = ('{:.' + str(precision) + 'g} {}').format(count_, unit)
if colored:
if years > 1e5:
# Blue is VERY safe
ret = ub.color_text(ret, 'blue')
elif years > 80:
# Green is safe
ret = ub.color_text(ret, 'green')
elif years > 10:
# Yellow is short-term safe
ret = ub.color_text(ret, 'yellow')
else:
# Red is not safe
ret = ub.color_text(ret, 'red')
return ret
def prepare_solver(harn, force=False):
if force or not harn.solver_fpath:
harn.train.prepare_input()
harn.train.prepare_gtstats(harn.task)
gtstats = harn.train.gtstats
class_weights = gtstats.loss_weight.loc[harn.task.classnames]
class_weights = class_weights.drop(harn.task.ignore_classnames)
print(ub.color_text('[segnet] Preparing solver', 'blue'))
harn.train_base = ub.ensuredir((harn.archdir, 'train'))
if harn.init_pretrained_fpath is not None:
if harn.train_init_id is None:
harn.train_init_id = util.hash_file(
harn.init_pretrained_fpath)[:harn.abbrev]
else:
harn.train_init_id = None
harn.train_hyper_id = util.hash_data(
harn.params.hyper_id())[:harn.abbrev]
harn.train_id = '{}_{}_{}_{}'.format(harn.train.input_id,
harn.arch, harn.train_init_id,
harn.train_hyper_id)
print('+=========')
print('harn.hyper_strid = {!r}'.format(harn.params.hyper_id()))
print('harn.train_init_id = {!r}'.format(harn.train_init_id))
print('harn.arch = {!r}'.format(harn.arch))
print('harn.train_hyper_id = {!r}'.format(harn.train_hyper_id))
print('harn.train_id = {!r}'.format(harn.train_id))
print('+=========')
harn.solver_dpath = ub.ensuredir(
(harn.train_base, 'input_' + harn.train.input_id,
'solver_{}'.format(harn.train_id)))
harn.solver_fpath = models.make_solver_file(
harn.train.input_fpath,
arch=harn.arch,
dpath=harn.solver_dpath,
params=harn.params,
gpu_num=harn.gpu_num,
modelkw={
'batch_size': harn.train_batch_size,
'class_weights': class_weights,
'n_classes': len(harn.task.classnames),
'ignore_label': harn.task.ignore_label,
},
)
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False):
"""
issues a command on a repo
Example:
>>> # DISABLE_DOCTEST
>>> repo = dirname(dirname(ub.__file__))
>>> command = 'git status'
>>> sudo = False
>>> result = repocmd(repo, command, sudo)
>>> print(result)
"""
WIN32 = sys.platform.startswith('win32')
if WIN32:
assert not sudo, 'cant sudo on windows'
if command == 'short_status':
return repo.short_status()
command_list = [command]
cmdstr = '\n '.join([cmd_ for cmd_ in command_list])
if not dry:
import ubelt as ub
print('+--- *** repocmd(%s) *** ' % (cmdstr,))
print('repo=%s' % ub.color_text(repo.dpath, 'yellow'))
verbose = True
with repo.chdir_context():
ret = None
for count, command in enumerate(command_list):
if dry:
print(command)
continue
if not sudo or WIN32:
cmdinfo = ub.cmd(command, verbose=1)
out, err, ret = ub.take(cmdinfo, ['out', 'err', 'ret'])
else:
out, err, ret = ub.cmd('sudo ' + command)
if verbose > 1:
print('ret(%d) = %r' % (count, ret,))
if ret != 0:
if error == 'raise':
raise Exception('Failed command %r' % (command,))
elif error == 'return':
return out
else:
raise ValueError('unknown flag error=%r' % (error,))
if return_out:
return out
if not dry:
print('L____')
def gitcmd(repo, command):
try:
import ubelt as ub
except ImportError:
print()
print("************")
try:
print('repo=%s' % ub.color_text(repo.dpath, 'yellow'))
except Exception:
print('repo = %r ' % (repo,))
os.chdir(repo)
if command.find('git') != 0 and command != 'gcwip':
command = 'git ' + command
os.system(command)
print("************")
else:
repo_ = Repo(dpath=repo)
repo_.issue(command)
def ensure(repo, dry=False):
"""
Ensure that the repo is checked out on your local machine, that the
correct branch is checked out, and the upstreams are targeting the
correct remotes.
"""
if repo.verbose > 0:
if dry:
repo.debug(ub.color_text('Checking {}'.format(repo), 'blue'))
else:
repo.debug(ub.color_text('Ensuring {}'.format(repo), 'blue'))
if not exists(repo.dpath):
repo.debug('NEED TO CLONE {}'.format(repo))
if dry:
return
repo.ensure_clone()
repo._assert_clean()
# Ensure all registered remotes exist
for remote_name, remote_url in repo.remotes.items():
try:
remote = repo.pygit.remotes[remote_name]
have_urls = list(remote.urls)
if remote_url not in have_urls:
print('WARNING: REMOTE NAME EXIST BUT URL IS NOT {}. '
'INSTEAD GOT: {}'.format(remote_url, have_urls))
except (IndexError):
try:
print('NEED TO ADD REMOTE {}->{} FOR {}'.format(
remote_name, remote_url, repo))
if not dry:
repo._cmd('git remote add {} {}'.format(
remote_name, remote_url))
except ShellException:
if remote_name == repo.remote:
# Only error if the main remote is not available
raise
# Ensure we have the right remote
try:
remote = repo.pygit.remotes[repo.remote]
except IndexError:
if not dry:
raise AssertionError('Something went wrong')
else:
remote = None
if remote is not None:
try:
if not remote.exists():
raise IndexError
else:
repo.debug(
'The requested remote={} name exists'.format(remote))
except IndexError:
repo.debug('WARNING: remote={} does not exist'.format(remote))
else:
if remote.exists():
repo.debug('Requested remote does exists')
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
repo.info(
'Branch name not found in local remote. Attempting to fetch'
)
if dry:
repo.info('dry run, not fetching')
else:
repo._cmd('git fetch {}'.format(remote.name))
repo.info('Fetch was successful')
else:
repo.debug('Requested remote does NOT exist')
# Ensure the remote points to the right place
if repo.url not in list(remote.urls):
repo.debug(
'WARNING: The requested url={} disagrees with remote urls={}'
.format(repo.url, list(remote.urls)))
if dry:
repo.info('Dry run, not updating remote url')
else:
repo.info('Updating remote url')
repo._cmd('git remote set-url {} {}'.format(
repo.remote, repo.url))
# Ensure we are on the right branch
if repo.branch != repo.pygit.active_branch.name:
repo.debug('NEED TO SET BRANCH TO {} for {}'.format(
repo.branch, repo))
try:
repo._cmd('git checkout {}'.format(repo.branch))
except ShellException:
repo.debug(
'Checkout failed. Branch name might be ambiguous. Trying again'
)
try:
repo._cmd('git checkout -b {} {}/{}'.format(
repo.branch, repo.remote, repo.branch))
except ShellException:
raise Exception('does the branch exist on the remote?')
tracking_branch = repo.pygit.active_branch.tracking_branch()
if tracking_branch is None or tracking_branch.remote_name != repo.remote:
repo.debug('NEED TO SET UPSTREAM FOR FOR {}'.format(repo))
try:
remote = repo.pygit.remotes[repo.remote]
if not remote.exists():
raise IndexError
except IndexError:
repo.debug(
'WARNING: remote={} does not exist'.format(remote))
else:
if remote.exists():
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
if dry:
repo.info(
'Branch name not found in local remote. Dry run, use ensure to attempt to fetch'
)
else:
repo.info(
'Branch name not found in local remote. Attempting to fetch'
)
repo._cmd('git fetch {}'.format(repo.remote))
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
raise Exception(
'Branch name still does not exist')
if not dry:
repo._cmd(
'git branch --set-upstream-to={remote}/{branch} {branch}'
.format(remote=repo.remote,
branch=repo.branch))
else:
repo.info('Would attempt to set upstream')
# Print some status
repo.debug(' * branch = {} -> {}'.format(
repo.pygit.active_branch.name,
repo.pygit.active_branch.tracking_branch(),
))
def benchmark_attribute_access():
"""
How fast are different methods of accessing attributes? Lets find out!
"""
instances = {
'simple': Simple(),
'complex': Complex(),
'slot_simple': SimpleWithSlots(),
'slot_complex': ComplexWithSlots(),
}
import ubelt as ub
ti = ub.Timerit(100000, bestof=500, verbose=1, unit='us')
# Do this twice, but keep the second measure
data = ub.AutoDict()
for selfname, self in instances.items():
print(ub.color_text('--- SELF = {} ---'.format(selfname), 'blue'))
subdata = data[selfname] = {}
for timer in ti.reset('self.attr1'):
with timer:
self.attr1
subdata[ti.label] = ti.min()
for timer in ti.reset('getattr(self, attr1)'):
with timer:
getattr(self, 'attr1')
subdata[ti.label] = ti.min()
attrs = ['attr1', 'attr2']
for attrname in attrs:
for timer in ti.reset('hasattr(self, {})'.format(attrname)):
with timer:
hasattr(self, attrname)
subdata[ti.label] = ti.min()
for timer in ti.reset('getattr(self, {}, None)'.format(attrname)):
with timer:
getattr(self, attrname, None)
subdata[ti.label] = ti.min()
if 'slot' not in selfname.lower():
for timer in ti.reset(
'self.__dict__.get({}, None)'.format(attrname)):
with timer:
self.__dict__.get(attrname, None)
subdata[ti.label] = ti.min()
for timer in ti.reset('try/except: self.attr2'):
with timer:
try:
x = self.attr2
except AttributeError:
x = None
subdata[ti.label] = ti.min()
for timer in ti.reset('try/except: self.attr1'):
with timer:
try:
x = self.attr1
except AttributeError:
x = None
subdata[ti.label] = ti.min()
del x
try:
import pandas as pd
df = pd.DataFrame(data) * 1e9
try:
from kwil.util.util_pandas import _to_string_monkey
print(_to_string_monkey(df, key='minima'))
except Exception:
print(df)
except ImportError:
print('no pandas')
print(ub.repr2(data, nl=2, precision=4))
def ensure(repo, dry=False):
"""
Ensure that the repo is checked out on your local machine, that the
correct branch is checked out, and the upstreams are targeting the
correct remotes.
"""
if repo.verbose > 0:
if dry:
repo.debug(ub.color_text('Checking {}'.format(repo), 'blue'))
else:
repo.debug(ub.color_text('Ensuring {}'.format(repo), 'blue'))
if not exists(repo.dpath):
repo.debug('NEED TO CLONE {}: {}'.format(repo, repo.url))
if dry:
return
repo.ensure_clone()
repo._assert_clean()
# Ensure we have the right remote
remote = repo._registered_remote(dry=dry)
if remote is not None:
try:
if not remote.exists():
raise IndexError
else:
repo.debug(
'The requested remote={} name exists'.format(remote))
except IndexError:
repo.debug('WARNING: remote={} does not exist'.format(remote))
else:
if remote.exists():
repo.debug('Requested remote does exists')
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
repo.info(
'Branch name not found in local remote. Attempting to fetch'
)
if dry:
repo.info('dry run, not fetching')
else:
repo._cmd('git fetch {}'.format(remote.name))
repo.info('Fetch was successful')
else:
repo.debug('Requested remote does NOT exist')
# Ensure the remote points to the right place
if repo.url not in list(remote.urls):
repo.debug(
ub.paragraph('''
'WARNING: The requested url={} disagrees with remote
urls={}
''').format(repo.url, list(remote.urls)))
if dry:
repo.info('Dry run, not updating remote url')
else:
repo.info('Updating remote url')
repo._cmd('git remote set-url {} {}'.format(
repo.remote, repo.url))
# Ensure we are on the right branch
try:
active_branch_name = repo.pygit.active_branch.name
except TypeError:
# We may be on a tag, not a branch
candidates = [
tag for tag in repo.pygit.tags if tag.name == repo.branch
]
if len(candidates) != 1:
raise
else:
# branch is actually a tag
assert len(candidates) == 1
want_tag = candidates[0]
is_on_correct_commit = (repo.pygit.head.commit.hexsha ==
want_tag.commit.hexsha)
ref_is_tag = True
else:
ref_is_tag = False
tracking_branch = repo.pygit.active_branch.tracking_branch()
is_on_correct_commit = repo.branch == active_branch_name
if not is_on_correct_commit:
repo.debug('NEED TO SET BRANCH TO {} for {}'.format(
repo.branch, repo))
if dry:
repo.info('Dry run, not setting branch')
else:
try:
repo._cmd('git checkout {}'.format(repo.branch))
except ShellException:
repo.debug(
'Checkout failed. Branch name might be ambiguous. Trying again'
)
try:
repo._cmd('git fetch {}'.format(remote.name))
repo._cmd('git checkout -b {} {}/{}'.format(
repo.branch, repo.remote, repo.branch))
except ShellException:
raise Exception(
'does the branch exist on the remote?')
if not ref_is_tag:
if tracking_branch is None or tracking_branch.remote_name != repo.remote:
repo.debug('NEED TO SET UPSTREAM FOR FOR {}'.format(repo))
try:
remote = repo.pygit.remotes[repo.remote]
if not remote.exists():
raise IndexError
except IndexError:
repo.debug(
'WARNING: remote={} does not exist'.format(remote))
else:
if remote.exists():
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
if dry:
repo.info(
'Branch name not found in local remote. Dry run, use ensure to attempt to fetch'
)
else:
repo.info(
'Branch name not found in local remote. Attempting to fetch'
)
repo._cmd('git fetch {}'.format(
repo.remote))
remote_branchnames = [
ref.remote_head for ref in remote.refs
]
if repo.branch not in remote_branchnames:
raise Exception(
'Branch name still does not exist')
if not dry:
repo._cmd(
'git branch --set-upstream-to={remote}/{branch} {branch}'
.format(remote=repo.remote,
branch=repo.branch))
else:
repo.info('Would attempt to set upstream')
# Check if the current head is tagged
head_tags = [
tag for tag in repo.pygit.tags
if tag.commit.hexsha == repo.pygit.head.commit.hexsha
]
# Print some status
try:
repo.debug(' * branch = {} -> {}'.format(
repo.pygit.active_branch.name,
repo.pygit.active_branch.tracking_branch(),
))
except Exception:
pass
if head_tags:
repo.debug(' * head_tags = {}'.format(head_tags))
def run_xval_evaluation(task,
arch='segnet_proper',
pretrained=None,
hyperparams={},
xval_method='predef',
test_keys=None,
fit=True):
"""
Writes test/train data files containing the image paths that will be
used for testing and training as well as the appropriate solvers and
prediction models.
Args:
fit (bool): if False, we will only evaluate models that have been
trained so far (useful for getting results from existing while a
model is not done training)
CommandLine:
export PYTHONPATH=$HOME/code/fletch/build-py3/install/lib/python3.5/site-packages:$PYTHONPATH
python -m clab.tasks DivaV1.run_xval_evaluation
python -m clab.tasks DivaV1.run_xval_evaluation --batch-size=1
python -m clab.tasks DivaV1.run_xval_evaluation
rsync -avpr aretha:sseg/sseg-data/xval-solvers ~/sseg/sseg-data/xval-solvers
Script:
>>> from clab.tasks import *
>>> import clab
>>> task = DivaV1(clean=2)
>>> arch = 'segnet_proper'
>>> pretrained = '/home/local/KHQ/jon.crall/store/segnet-exact/SegNet-Tutorial/Models/Training/segnet_iter_30000.caffemodel'
>>> pretrained = 'segnet_proper_camvid.caffemodel'
>>> hyperparams = {'freeze_before': -23, 'max_iter': 10000}
>>> # Use external dataset to increase the amount of training data
>>> tutorial_dir = './SegNet-Tutorial'
>>> task.extend_data_from(clab.tasks.CamVid(tutorial_dir))
>>> task.run_xval_evaluation()
"""
from clab import harness
# from clab import models
xval_base = abspath(task.exptdir('xval'))
xval_results = []
for idx, xval_split in enumerate(
task.xval_splits(xval_method, test_keys=test_keys)):
print(ub.color_text('XVAL iteration {}'.format(idx), 'blue'))
# harn = task.harness_from_xval(idx, arch=arch,
# pretrained=pretrained,
# hyperparams=hyperparams,
# xval_method=xval_method,
# test_keys=test_keys)
# (train_data, test_data) = xval_split
xval_dpath = ub.ensuredir((xval_base, 'split_{:0=2}'.format(idx)))
(train, test) = xval_split
harn = harness.Harness(workdir=xval_dpath, arch=arch)
harn.set_inputs(train, test)
# harn.train.im_paths = train_data[0]
# harn.train.gt_paths = train_data[1]
# harn.test.im_paths = test_data[0]
# harn.test.gt_paths = test_data[1]
# assert len(harn.train.im_paths) > 0
# assert len(harn.train.gt_paths) > 0
# assert len(harn.test.im_paths) > 0
# assert len(harn.test.gt_paths) > 0
harn.params.update(hyperparams)
harn.init_pretrained_fpath = pretrained
harn.test.prepare_images()
harn.train.prepare_images()
harn.gpu_num = gpu_util.find_unused_gpu(min_memory=6000)
print('harn.gpu_num = {!r}'.format(harn.gpu_num))
if harn.gpu_num is not None:
avail_mb = gpu_util.gpu_info()[harn.gpu_num]['mem_avail']
# Estimate how much we can fit in memory
# TODO: estimate this from the model arch instead.
# (90 is a mgic num corresponding to segnet_proper)
harn.train_batch_size = int(
(avail_mb * 90) // np.prod(task.input_shape))
harn.train_batch_size = int(harn.train_batch_size)
if harn.train_batch_size == 0:
raise MemoryError(
'not enough GPU memory to train the model')
else:
# not sure what the best to do on CPU is. Probably nothing.
harn.train_batch_size = 4
harn.prepare_solver()
# Check if we can resume a previous training state
print(
ub.color_text('Checking for previous snapshot states', 'blue'))
previous_states = harn.snapshot_states()
print('Found {} previous_states'.format(len(previous_states)))
from clab.backend import iface_caffe as iface
solver_info = iface.parse_solver_info(harn.solver_fpath)
prev_iter = 0
if previous_states:
prev_state = previous_states[-1]
prev_iter = iface.snapshot_iterno(prev_state)
if prev_iter == 0:
if fit:
print(
ub.color_text('Starting a fresh training session',
'blue'))
# harn.fit()
list(harn.fit2())
else:
print(
ub.color_text('Would start a fresh training session',
'yellow'))
elif prev_iter < solver_info['max_iter']:
if fit:
# continue from the previous iteration
print(
ub.color_text(
'Resume training from iter {}'.format(prev_iter),
'blue'))
# harn.fit(prevstate_fpath=prev_state)
list(harn.fit2(prevstate_fpath=prev_state))
else:
print(
ub.color_text(
'Would resume training from iter {}'.format(
prev_iter), 'yellow'))
else:
print(
ub.color_text('Already finished training this model',
'blue'))
for _ in harn.deploy_trained_for_testing():
# hack to evaulate while deploying
harn.evaulate_all()
xval_results.append(list(harn._test_results_fpaths()))
return xval_results
def postprocess_training_weights(harn,
train_weights_path=None,
force=False):
"""
Fixes BN params in the weights file for testing
Ideally this should be run after dumping a snapshot to disk
(because it requires training data to compute the BN params)
However, you can run it before testing IF you still know where all the
training data is.
"""
solver_info = iface.parse_solver_info(harn.solver_fpath)
train_model_path = solver_info['train_model_path']
if train_weights_path is None:
# Get the most recent set of weights
snapshot_models = iface.load_snapshot_weight_paths(
solver_info['snapshot_prefix'])
train_weights_path = snapshot_models[-1]
iterno = iface.snapshot_iterno(train_weights_path)
test_weight_suffix = ('{}_{:08d}'.format(harn.train_id, iterno))
harn.test_weights_dir = ub.ensuredir((
harn.solver_dpath,
'testable',
))
# Names of the files we are about to write
test_deploy_fpath = join(
harn.test_weights_dir,
'deploy_{}.prototxt'.format(test_weight_suffix))
test_weights_fpath = join(
harn.test_weights_dir,
'test_weights_{}.caffemodel'.format(test_weight_suffix))
BN_calc_path = join(
harn.test_weights_dir,
'__temp_bn_stats_{}.prototext'.format(test_weight_suffix))
if exists(test_weights_fpath) and not force:
# lazy cache
harn.test_weights_fpath = test_weights_fpath
return test_weights_fpath
print(ub.color_text('Postprocessing BN stats', 'blue'))
print('Building testable weights from train_model_path = {!r}'.format(
train_model_path))
print(
'Testable weights will be saved to harn.test_weights_fpath = {!r}'.
format(harn.test_weights_fpath))
harn.prepare_workdir()
from pysseg.backend import batch_norm_stats
from google.protobuf import text_format
train_net = batch_norm_stats.make_testable(train_model_path)
with open(BN_calc_path, 'w') as f:
f.write(text_format.MessageToString(train_net))
# use testable net to calculate BN layer stats
# these must be TRAIN datas
train_im_paths, train_gt_paths = batch_norm_stats.extract_dataset(
train_net)
# HACK: remove aug
train_gt_paths = [
p for p in train_gt_paths if '_aug' not in basename(p)
]
train_im_paths = [
p for p in train_im_paths if '_aug' not in basename(p)
]
# HACK: remove part-scale2
train_gt_paths = [
p for p in train_gt_paths if 'part-scale2' not in basename(p)
]
train_im_paths = [
p for p in train_im_paths if 'part-scale2' not in basename(p)
]
# This does a single forward pass on the testing dataset so we can get
# some stats
gpu_num = harn.gpu_num
test_net, test_msg = batch_norm_stats.make_test_files(
BN_calc_path, train_weights_path, train_im_paths, gpu_num=gpu_num)
# save deploy prototxt
# (note deploy is a prediction model that is meant to be used with blob
# inputs)
with open(test_deploy_fpath, 'w') as f:
f.write(text_format.MessageToString(test_msg))
test_net.save(test_weights_fpath)
harn.test_weights_fpath = test_weights_fpath
return test_weights_fpath
def color_text(text):
ub.color_text(text)
def main():
# TODO: find a better place for root
ROOT = join(os.getcwd())
# ROOT = '.'
os.chdir(ROOT)
NAME = 'pyhesaff'
VERSION = '0.1.2'
DOCKER_TAG = '{}-{}'.format(NAME, VERSION)
QUAY_REPO = 'quay.io/erotemic/manylinux-for'
DOCKER_URI = '{QUAY_REPO}:{DOCKER_TAG}'.format(**locals())
dockerfile_fpath = join(ROOT, 'Dockerfile')
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
"""
Notes:
docker run --rm -it quay.io/pypa/manylinux2010_x86_64 /bin/bash
---
ls /opt/python
"""
BASE_IMAGE = 'quay.io/pypa/manylinux2010_x86_64'
docker_code = ub.codeblock(f'''
FROM {BASE_IMAGE}
RUN yum install lz4-devel -y
RUN MB_PYTHON_TAG=cp27-cp27m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp27-cp27mu && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp35-cp35m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp36-cp36m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp37-cp37m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp38-cp38 && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
''')
docker_code2 = '\n\n'.join(
[ub.paragraph(p) for p in docker_code.split('\n\n')])
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code2, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code2)
docker_build_cli = ' '.join([
'docker',
'build',
'--tag {}'.format(DOCKER_TAG),
'-f {}'.format(dockerfile_fpath),
'.',
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
if ub.argflag('--dry'):
print('DRY RUN')
print('WOULD RUN')
print(docker_build_cli)
else:
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(
info['ret']))
else:
print(ub.color_text('\n--- SUCCESS ---', 'green'))
print(
ub.highlight_code(
ub.codeblock(r'''
# Finished creating the docker image.
# To test / export / publish you can do something like this:
# Test that we can get a bash terminal
docker run -it {DOCKER_TAG} /bin/bash
# Create a tag for the docker image
docker tag {DOCKER_TAG} {DOCKER_URI}
# Export your docker image to a file
docker save -o ${ROOT}/{DOCKER_TAG}.docker.tar {DOCKER_TAG}
# Login to a docker registry (we are using quay)
# In some cases this works,
docker login
# But you may need to specify secret credentials
load_secrets
echo "QUAY_USERNAME = $QUAY_USERNAME"
docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io
unload_secrets
# Upload the docker image to quay.io
docker push {DOCKER_URI}
''').format(
NAME=NAME,
ROOT=ROOT,
DOCKER_TAG=DOCKER_TAG,
DOCKER_URI=DOCKER_URI,
),
'bash',
))
PUBLISH = 0
if PUBLISH:
cmd1 = 'docker tag {DOCKER_TAG} {DOCKER_URI}'.format(**locals())
cmd2 = 'docker push {DOCKER_URI}'.format(**locals())
print('-- <push cmds> ---')
print(cmd1)
print(cmd2)
print('-- </push cmds> ---')
def main():
import os
ROOT = join(os.getcwd())
ROOT = ub.expandpath('~/code/hesaff')
os.chdir(ROOT)
VERSION = setup.version
PY_VER = sys.version_info.major
NAME = 'pyhesaff'
tag = '{}-{}-py{}'.format(NAME, VERSION, PY_VER)
# context_dpath = ub.ensuredir((ROOT, 'docker/context'))
staging_dpath = ub.ensuredir((ROOT, 'docker/staging'))
# Prestage the multibuild repo
if not exists(join(staging_dpath, 'multibuild')):
# FIXME: make robust in the case this fails
info = ub.cmd(
'git clone https://github.com/matthew-brett/multibuild.git',
cwd=staging_dpath,
verbose=3)
if not exists(join(staging_dpath, 'opencv')):
# FIXME: make robust in the case this fails
opencv_version = '4.1.0'
fpath = ub.grabdata(
'https://github.com/opencv/opencv/archive/{}.zip'.format(
opencv_version),
verbose=1)
ub.cmd('ln -s {} .'.format(fpath), cwd=staging_dpath, verbose=3)
ub.cmd('unzip {}'.format(fpath), cwd=staging_dpath, verbose=3)
import shutil
shutil.move(join(staging_dpath, 'opencv-' + opencv_version),
join(staging_dpath, 'opencv'))
stage_self(ROOT, staging_dpath)
dockerfile_fpath = join(ROOT, 'Dockerfile')
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
docker_code = ub.codeblock('''
FROM quay.io/skvark/manylinux1_x86_64
# SETUP ENV
ARG MB_PYTHON_VERSION=3.6
ARG ENABLE_CONTRIB=0
ARG ENABLE_HEADLESS=1
ENV PYTHON_VERSION=3.6
ENV PYTHON_ROOT=/opt/python/cp36-cp36m/
ENV PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages/
ENV PATH=/opt/python/cp36-cp36m/bin:$PATH
ENV PYTHON_EXE=/opt/python/cp36-cp36m/python
ENV MULTIBUILD_DIR=/root/code/multibuild
ENV HOME=/root
# params to bdist_wheel. used to set osx build target.
ENV TEST_DEPENDS="numpy==1.11.1"
ENV BDIST_PARAMS=""
ENV USE_CCACHE=1
ENV PLAT=x86_64
ENV UNICODE_WIDTH=32
# -e BUILD_COMMANDS="$build_cmds" \
# -e PYTHON_VERSION="$MB_PYTHON_VERSION" \
# -e UNICODE_WIDTH="$UNICODE_WIDTH" \
# -e BUILD_COMMIT="$BUILD_COMMIT" \
# -e CONFIG_PATH="$CONFIG_PATH" \
# -e ENV_VARS_PATH="$ENV_VARS_PATH" \
# -e WHEEL_SDIR="$WHEEL_SDIR" \
# -e MANYLINUX_URL="$MANYLINUX_URL" \
# -e BUILD_DEPENDS="$BUILD_DEPENDS" \
# -e USE_CCACHE="$USE_CCACHE" \
# -e REPO_DIR="$repo_dir" \
# -e PLAT="$PLAT" \
# These are defined in the parent image
# ENV JPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include
# ENV JPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a
RUN mkdir -p /io
WORKDIR /root
# Setup code / scripts
COPY docker/staging/multibuild /root/code/multibuild
# Hack to fix issue
RUN find $MULTIBUILD_DIR -iname "*.sh" -type f -exec sed -i 's/gh-clone/gh_clone/g' {} +
# Copy bash configs (mirrors the environs)
COPY docker/config.sh /root/config.sh
COPY docker/bashrc.sh /root/.bashrc
# Setup a virtualenv
RUN source /root/.bashrc && \
$PYTHON_EXE -m pip install --upgrade pip && \
$PYTHON_EXE -m pip install virtualenv && \
$PYTHON_EXE -m virtualenv --python=$PYTHON_EXE $HOME/venv
# Install packages in virtual environment
RUN source /root/.bashrc && \
pip install cmake ninja scikit-build wheel numpy
# This is very different for different operating systems
# https://github.com/skvark/opencv-python/blob/master/setup.py
COPY docker/staging/opencv /root/code/opencv
RUN source /root/.bashrc && \
source code/multibuild/common_utils.sh && \
source code/multibuild/travis_linux_steps.sh && \
mkdir -p /root/code/opencv/build && \
cd /root/code/opencv/build && \
cmake -G "Unix Makefiles" \
-DINSTALL_CREATE_DISTRIB=ON \
-DOPENCV_SKIP_PYTHON_LOADER=ON \
-DBUILD_opencv_apps=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_PERF_TESTS=OFF \
-DBUILD_DOCS=OFF \
-DWITH_QT=OFF \
-DWITH_IPP=OFF \
-DWITH_V4L=ON \
-DBUILD_JPEG=OFF \
-DENABLE_PRECOMPILED_HEADERS=OFF \
-DJPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include \
-DJPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a \
/root/code/opencv
# Note: there is no need to compile the above with python
# -DPYTHON3_EXECUTABLE=$PYTHON_EXE \
# -DBUILD_opencv_python3=ON \
# -DOPENCV_PYTHON3_INSTALL_PATH=python \
RUN source /root/.bashrc && \
source code/multibuild/common_utils.sh && \
source code/multibuild/travis_linux_steps.sh && \
cd /root/code/opencv/build && \
make -j9 && make install
COPY docker/staging/hesaff /root/code/hesaff
# # Use skbuild to build hesaff
# RUN source /root/.bashrc && \
# cd /root/code/hesaff && \
# CMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" python setup.py build_ext --inplace
# export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
# Use cmake to build hesaff 9maybe not needed?)
# RUN source /root/.bashrc && \
# mkdir -p /root/code/hesaff/build && \
# cd /root/code/hesaff/build && \
# CXXFLAGS="-std=c++11 $CXXFLAGS" cmake -G "Unix Makefiles" /root/code/hesaff && \
# make
# Use skbuild to build hesaff
RUN source /root/.bashrc && \
cd /root/code/hesaff && \
python setup.py build && \
python setup.py bdist_wheel
# RUN source /root/.bashrc && \
# pip install xdoctest
''')
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code)
docker_build_cli = ' '.join([
'docker',
'build',
# '--build-arg PY_VER={}'.format(PY_VER),
'--tag {}'.format(tag),
'-f {}'.format(dockerfile_fpath),
'.'
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(
info['ret']))
else:
print(ub.color_text('\n--- SUCCESS ---', 'green'))
DEPLOY = True
if DEPLOY:
VMNT_DIR = '{ROOT}/{NAME}-docker/vmnt'.format(NAME=NAME, ROOT=ROOT)
print('VMNT_DIR = {!r}'.format(VMNT_DIR))
ub.ensuredir(VMNT_DIR)
# TODO: Correctly mangle the ffmpeg libs as done via
# ls -a ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/
# ls ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/.libs
# cp code/hesaff/build/libhesaff.so /root/vmnt
# cp /root/ffmpeg_build/lib/libavcodec.so.58 /root/vmnt
# cp /root/ffmpeg_build/lib/libavformat.so.58 /root/vmnt
# cp /root/ffmpeg_build/lib/libavutil.so.56 /root/vmnt
# cp /root/ffmpeg_build/lib/libswscale.so.5 /root/vmnt
inside_cmds = ' && '.join(
ub.codeblock('''
cp code/hesaff/dist/pyhesaff*.whl /root/vmnt
''').split('\n'))
docker_run_cli = ' '.join([
'docker', 'run', '-v {}:/root/vmnt/'.format(VMNT_DIR), '-it', tag,
'bash -c "{}"'.format(inside_cmds)
])
print(docker_run_cli)
info = ub.cmd(docker_run_cli, verbose=3)
assert info['ret'] == 0
# import shutil
# PKG_DIR = join(ROOT, 'pyhesaff')
# shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libhesaff-manylinux1_x86_64.so'))
# TODO: do this correctly
# shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libavcodec.so.58'))
# shutil.copy(join(VMNT_DIR, 'libavformat.so.58'), join(PKG_DIR, 'libavformat.so.58'))
# shutil.copy(join(VMNT_DIR, 'libavutil.so.56'), join(PKG_DIR, 'libavutil.so.56'))
# shutil.copy(join(VMNT_DIR, 'libswscale.so.5'), join(PKG_DIR, 'libswscale.so.5'))
# print(ub.highlight_code(ub.codeblock(
print(
ub.highlight_code(
ub.codeblock(r'''
# Finished creating the docker image.
# To test / export you can do something like this:
VMNT_DIR={ROOT}/{NAME}-docker/vmnt
mkdir -p VMNT_DIR
TAG={tag}
# Test that we can get a bash terminal
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash
# Move deployment to the vmnt directory
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 -m xdoctest pyhesaff'
# Run system tests
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 run_doctests.sh'
# Inside bash test that we can fit a new model
python -m pyhessaff demo
mkdir -p ${ROOT}/{NAME}-docker/dist
docker save -o ${ROOT}/{NAME}-docker/dist/{tag}.docker.tar {tag}
''').format(NAME=NAME, ROOT=ROOT, tag=tag), 'bash'))
def color(x):
if x > 500:
return ub.color_text(str(x), 'green')
else:
return ub.color_text(str(x), 'red')
def __repr__(self):
return ub.color_text('<AnInstance>', 'yellow')
def _entropy_refine(depth, nodes, jdxs):
"""
Recursively descend the class tree starting at the coursest level.
At each level we decide if the items will take a category at this
level of granulatority or try to take a more fine-grained label.
Args:
depth (int): current depth in the tree
nodes (list) : set of sibling nodes at a this level
jdxs (ArrayLike): item indices that made it to this level (note
idxs are used for class indices)
"""
if DEBUG:
print(ub.color_text('* REFINE nodes={}'.format(nodes), 'blue'))
# Look at the probabilities of each node at this level
idxs = sorted(self.node_to_idx[node] for node in nodes)
if ignore_class_idxs:
ignore_nodes = set(ub.take(self.idx_to_node, ignore_class_idxs))
idxs = sorted(set(idxs) - set(ignore_class_idxs))
if len(idxs) == 0:
raise ValueError('Cannot ignore all top-level classes')
probs = flat_class_probs[jdxs][:, idxs]
# Choose a highest probability category to predict at this level
pred_conf, pred_cx = impl.max_argmax(probs, axis=1)
pred_idxs = np.array(idxs)[impl.numpy(pred_cx)]
# Group each example which predicted the same class at this level
group_idxs, groupxs = kwarray.group_indices(pred_idxs)
if DEBUG:
groupxs = list(ub.take(groupxs, group_idxs.argsort()))
group_idxs = group_idxs[group_idxs.argsort()]
# print('groupxs = {!r}'.format(groupxs))
# print('group_idxs = {!r}'.format(group_idxs))
for idx, groupx in zip(group_idxs, groupxs):
# Get the children of this node (idx)
node = self.idx_to_node[idx]
children = sorted(self.graph.successors(node))
if ignore_class_idxs:
children = sorted(set(children) - ignore_nodes)
if children:
# Check if it would be simple to refine the coarse category
# current prediction into one of its finer-grained child
# categories. Do this by considering the entropy at this
# level if we replace this coarse-node with the child
# fine-nodes. Then compare that entropy to what we would
# get if we were perfectly uncertain about the child node
# prediction (i.e. the worst case). If the entropy we get
# is much lower than the worst case, then it is simple to
# descend the tree and predict a finer-grained label.
# Expand this node into all of its children
child_idxs = set(self.node_to_idx[child] for child in children)
# Get example indices (jdxs) assigned to category idx
groupx.sort()
group_jdxs = jdxs[groupx]
# Expand this parent node, but keep the parent's siblings
ommer_idxs = sorted(set(idxs) - {idx}) # Note: ommer = Aunt/Uncle
expanded_idxs = sorted(ommer_idxs) + sorted(child_idxs)
expanded_probs = flat_class_probs[group_jdxs][:, expanded_idxs]
# Compute the entropy of the expanded distribution
h_expanded = _criterion(expanded_probs)
# Probability assigned to the parent
p_parent = flat_class_probs[group_jdxs][:, idx:idx + 1]
# Get the absolute probabilities assigned the parents siblings
ommer_probs = flat_class_probs[group_jdxs][:, sorted(ommer_idxs)]
# Compute the worst-case entropy after expanding the node
# In the worst case the parent probability is distributed
# uniformly among all of its children
c = len(children)
child_probs_worst = impl.tile(p_parent / c, reps=[1, c])
expanded_probs_worst = impl.hstack([ommer_probs, child_probs_worst])
h_expanded_worst = _criterion(expanded_probs_worst)
# Normalize the entropy we got by the worst case.
# eps = float(np.finfo(np.float32).min)
eps = 1e-30
complexity_ratio = h_expanded / (h_expanded_worst + eps)
simplicity_ratio = 1 - complexity_ratio
# If simplicity ratio is over a threshold refine the parent
refine_flags = simplicity_ratio > thresh
if always_refine_idxs is not None:
if idx in always_refine_idxs:
refine_flags[:] = 1
if len(child_idxs) == 1:
# hack: always refine when there is one child, in this
# case the simplicity measure will always be zero,
# which is likely a problem with this criterion.
refine_flags[:] = 1
refine_flags = kwarray.ArrayAPI.numpy(refine_flags).astype(np.bool)
if DEBUG:
print('-----------')
print('idx = {!r}'.format(idx))
print('node = {!r}'.format(self.idx_to_node[idx]))
print('ommer_idxs = {!r}'.format(ommer_idxs))
print('ommer_nodes = {!r}'.format(
list(ub.take(self.idx_to_node, ommer_idxs))))
print('depth = {!r}'.format(depth))
import pandas as pd
print('expanded_probs =\n{}'.format(
ub.repr2(expanded_probs, precision=2,
with_dtype=0, supress_small=True)))
df = pd.DataFrame({
'h': h_expanded,
'h_worst': h_expanded_worst,
'c_ratio': complexity_ratio,
's_ratio': simplicity_ratio,
'flags': refine_flags.astype(np.uint8)
})
print(df)
print('-----------')
if np.any(refine_flags):
refine_jdxs = group_jdxs[refine_flags]
refine_idxs, refine_conf = _entropy_refine(depth + 1, children, refine_jdxs)
# Overwrite course decisions with refined decisions.
refine_groupx = groupx[refine_flags]
pred_idxs[refine_groupx] = refine_idxs
pred_conf[refine_groupx] = refine_conf
return pred_idxs, pred_conf
def apply(registery, funcname, num_workers=0, **kwargs):
print(ub.color_text('--- APPLY {} ---'.format(funcname), 'white'))
print(' * num_workers = {!r}'.format(num_workers))
if num_workers == 0:
processed_repos = []
for repo in registery.repos:
print(ub.color_text('--- REPO = {} ---'.format(repo), 'blue'))
try:
getattr(repo, funcname)(**kwargs)
except DirtyRepoError:
print(
ub.color_text('Ignoring dirty repo={}'.format(repo),
'red'))
processed_repos.append(repo)
else:
from concurrent import futures
# with futures.ThreadPoolExecutor(max_workers=num_workers) as pool:
with futures.ProcessPoolExecutor(max_workers=num_workers) as pool:
tasks = []
for i, repo in enumerate(registery.repos):
future = pool.submit(worker, repo, funcname, kwargs)
future.repo = repo
tasks.append(future)
processed_repos = []
for future in futures.as_completed(tasks):
repo = future.repo
print(
ub.color_text('--- REPO = {} ---'.format(repo),
'blue'))
try:
repo = future.result()
except DirtyRepoError:
print(
ub.color_text(
'Ignoring dirty repo={}'.format(repo), 'red'))
else:
print(repo._getlogs())
processed_repos.append(repo)
print(
ub.color_text('--- FINISHED APPLY {} ---'.format(funcname),
'white'))
SHOW_CMDLOG = 1
if SHOW_CMDLOG:
print('LOGGED COMMANDS')
import os
ORIG_CWD = MY_CWD = os.getcwd()
for repo in processed_repos:
print('# --- For repo = {!r} --- '.format(repo))
for t in repo._logged_cmds:
cmd, cwd = t
if cwd is None:
cwd = os.get_cwd()
if cwd != MY_CWD:
print('cd ' + ub.shrinkuser(cwd))
MY_CWD = cwd
print(cmd)
print('cd ' + ub.shrinkuser(ORIG_CWD))
def submit_query_request(
qreq_,
use_cache=None,
use_bigcache=None,
verbose=None,
save_qcache=None,
use_supercache=None,
invalidate_supercache=None,
):
"""
Called from qreq_.execute
Checks a big cache for qaid2_cm. If cache miss, tries to load each cm
individually. On an individual cache miss, it preforms the query.
CommandLine:
python -m wbia.algo.hots.match_chips4 --test-submit_query_request
Examples:
>>> # SLOW_DOCTEST
>>> # xdoctest: +SKIP
>>> from wbia.algo.hots.match_chips4 import * # NOQA
>>> import wbia
>>> qaid_list = [1]
>>> daid_list = [1, 2, 3, 4, 5]
>>> use_bigcache = True
>>> use_cache = True
>>> ibs = wbia.opendb(db='testdb1')
>>> qreq_ = ibs.new_query_request(qaid_list, daid_list, verbose=True)
>>> cm_list = submit_query_request(qreq_=qreq_)
"""
# Get flag defaults if necessary
if verbose is None:
verbose = pipeline.VERB_PIPELINE
if use_cache is None:
use_cache = USE_CACHE
if save_qcache is None:
save_qcache = SAVE_CACHE
if use_bigcache is None:
use_bigcache = USE_BIGCACHE
if use_supercache is None:
use_supercache = USE_SUPERCACHE
# Create new query request object to store temporary state
if verbose:
# logger.info('[mc4] --- Submit QueryRequest_ --- ')
logger.info(
ub.color_text('[mc4] --- Submit QueryRequest_ --- ', 'yellow'))
assert qreq_ is not None, 'query request must be prebuilt'
# Check fo empty queries
try:
assert len(qreq_.daids) > 0, 'there are no database chips'
assert len(qreq_.qaids) > 0, 'there are no query chips'
except AssertionError as ex:
ut.printex(
ex,
'Impossible query request',
iswarning=True,
keys=['qreq_.qaids', 'qreq_.daids'],
)
if ut.SUPER_STRICT:
raise
cm_list = [None for qaid in qreq_.qaids]
else:
# --- BIG CACHE ---
# Do not use bigcache single queries
is_big = len(qreq_.qaids) > MIN_BIGCACHE_BUNDLE
use_bigcache_ = use_bigcache and use_cache and is_big
if use_bigcache_ or save_qcache:
cacher = qreq_.get_big_cacher()
if use_bigcache_:
try:
qaid2_cm = cacher.load()
cm_list = [qaid2_cm[qaid] for qaid in qreq_.qaids]
except (IOError, AttributeError):
pass
else:
return cm_list
# ------------
# Execute query request
qaid2_cm = execute_query_and_save_L1(
qreq_,
use_cache,
save_qcache,
verbose=verbose,
use_supercache=use_supercache,
invalidate_supercache=invalidate_supercache,
)
# ------------
if save_qcache and is_big:
cacher.save(qaid2_cm)
cm_list = [qaid2_cm[qaid] for qaid in qreq_.qaids]
return cm_list
def lr_range_scan(harn, low=1e-6, high=10.0, num=8, niter_train=1,
niter_vali=0, scale='log'):
"""
This takes longer than lr_range_test, but is theoretically more robust.
Args:
low (float, default=1e-6): minimum lr to scan
high (float, default=1.0): maximum lr to scan
num (int, default=32): number of lrs to scan
niter_train (int, default=10): number of training batches to test for each lr
niter_vali (int, default=0): number of validation batches to test for each lr
Notes:
New better lr test:
* For each lr in your scan range
* reset model to original state
* run a few batches and backprop after each
* iterate over those batches again and compute their
loss, but dont backprop this time.
* associate the average of those losses with the learning rate.
Example:
>>> from netharn.prefit.lr_tests import *
>>> import netharn as nh
>>> harn = nh.FitHarn.demo().initialize()
>>> result = lr_range_scan(harn)
>>> print('result = {!r}'.format(result))
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> result.draw()
>>> kwplot.show_if_requested()
Ignore:
>>> import sys
>>> sys.path.append('/home/joncrall/code/netharn/examples')
>>> from mnist import setup_harn
>>> harn = setup_harn().initialize()
>>> harn.preferences['prog_backend'] = 'progiter'
>>> result = lr_range_scan(harn, niter_train=100, niter_vali=10, num=32)
>>> print('result = {!r}'.format(result))
>>> result.draw()
Ignore:
>>> from netharn.mixins import *
>>> import sys
>>> sys.path.append('/home/joncrall/code/netharn/examples')
>>> from ggr_matching import setup_harn
>>> harn = setup_harn(workers=6, xpu=0).initialize()
>>> result = lr_range_scan(harn, niter_train=6 * 2)
>>> print('recommended_lr = {!r}'.format(recommended_lr))
>>> draw()
TODO:
- [ ] ensure that this is a randomized sample of the validation
dataset.
- [ ] cache the dataset if it fits into memory after we run the first
epoch.
"""
import netharn as nh
use_vali = bool(niter_vali)
# These are the learning rates we will scan through
if scale == 'linear':
learning_rates = np.linspace(low, high, num=num,
endpoint=True)
elif scale == 'log':
log_b, base = np.log10, 10
learning_rates = np.logspace(log_b(low), log_b(high),
num=num, base=base, endpoint=True)
else:
raise KeyError(scale)
orig_model_state = copy.deepcopy(harn.model.state_dict())
orig_optim_state = copy.deepcopy(harn.optimizer.state_dict())
_orig_on_batch = harn.on_batch
_orig_on_epoch = harn.on_epoch
try:
failed_lr = None
best_lr = 0
best_loss = float('inf')
records = {}
records['train'] = ub.ddict(list)
if use_vali:
records['vali'] = ub.ddict(list)
# TODO: find a way to disable callabacks nicely
harn.on_batch = lambda *args, **kw: None
harn.on_epoch = lambda *args, **kw: None
prog = harn._make_prog(learning_rates, desc='scan learning rates',
disable=not harn.preferences['show_prog'],
total=len(learning_rates), leave=True,
dynamic_ncols=True, position=0)
harn.info('Running lr-scan')
for lr in prog:
# prog.set_description(ub.color_text('scan lr = {:.2g}'.format(lr), 'darkgreen'))
if hasattr(prog, 'ensure_newline'):
prog.ensure_newline()
# Reset model back to its original state
harn.optimizer.load_state_dict(orig_optim_state)
harn.model.load_state_dict(orig_model_state)
# Set the optimizer to the current lr we are scanning
harn.optimizer.param_groups[0]['lr'] = lr
# Run a partial training and validation epoch
try:
epoch_metrics = {}
epoch_metrics['train'] = harn._demo_epoch('train', learn=True, max_iter=niter_train)
curr_loss = epoch_metrics['train']['loss']
if curr_loss > 1000:
raise nh.fit_harn.TrainingDiverged
if use_vali:
epoch_metrics['vali'] = harn._demo_epoch('vali', learn=False, max_iter=niter_vali)
curr_loss = epoch_metrics['vali']['loss']
if curr_loss < best_loss:
best_lr = lr
best_loss = curr_loss
curr_text = ub.color_text('curr_lr={:.2g}, best_loss={:.2f}'.format(lr, curr_loss), 'green')
else:
curr_text = ub.color_text('curr_lr={:.2g}, best_loss={:.2f}'.format(lr, curr_loss), 'red')
if hasattr(prog, 'ensure_newline'):
prog.set_extra(
curr_text + ', ' +
ub.color_text('best_lr={:.2g}, best_loss={:.2f}'.format(best_lr, best_loss), 'white')
)
prog.update()
prog.ensure_newline()
except nh.fit_harn.TrainingDiverged:
harn.info('Learning is causing divergence, stopping lr-scan')
for tag in records.keys():
records[tag]['lr'].append(lr)
records[tag]['loss'].append(1000)
raise
for tag, epoch_metrics in epoch_metrics.items():
records[tag]['lr'].append(lr)
for key, value in epoch_metrics.items():
records[tag][key].append(value)
harn.info('Finished lr-scan')
except nh.fit_harn.TrainingDiverged:
failed_lr = lr
print('failed_lr = {!r}'.format(failed_lr))
except Exception:
failed_lr = lr
print('failed_lr = {!r}'.format(failed_lr))
raise
finally:
# Reset model back to its original state
harn.optimizer.load_state_dict(orig_optim_state)
harn.model.load_state_dict(orig_model_state)
harn.on_batch = _orig_on_batch
harn.on_epoch = _orig_on_epoch
# Choose an lr to recommend
tag = 'vali' if 'vali' in records else 'train'
if records.get(tag, []):
best_idx = ub.argmin(records[tag]['loss'])
best_lr = records[tag]['lr'][best_idx]
# Because we aren't doing crazy EWMAs and because we reset weights, we
# should be able to simply recommend the lr that minimized the loss.
# We may be able to do parabolic extrema finding to get an even better
# estimate in most cases.
recommended_lr = best_lr
else:
recommended_lr = None
# Give the user a way of visualizing what we did
def draw():
import kwplot
plt = kwplot.autoplt()
plotkw = dict(
xlabel='learning-rate', ylabel='loss', xscale=scale,
ymin=0, xmin='data', xmax=high, fnum=1,
)
R = 2 if 'vali' in records else 1
for i, tag in enumerate(['train', 'vali']):
if tag in records:
kwplot.multi_plot(
xdata=records[tag]['lr'], ydata=records[tag]['loss'],
ymax=1.2 * np.percentile(records[tag]['loss'], 60) / .6,
pnum=(1, R, i), title=tag + ' lr-scan', **plotkw)
plt.plot(best_lr, best_loss, '*')
result = TestResult(records, recommended_lr)
result.draw = draw
return result
def on_epoch(harn):
"""
Custom code executed at the end of each epoch.
This function can optionally return a dictionary containing any scalar
quality metrics that you wish to log and monitor. (Note these will be
plotted to tensorboard if that is installed).
Notes:
It is ok to do some medium lifting in this function because it is
run relatively few times.
Returns:
dict: dictionary of scalar metrics for netharn to log
CommandLine:
xdoctest -m netharn.examples.classification ClfHarn.on_epoch
Example:
>>> harn = setup_harn().initialize()
>>> harn._demo_epoch('vali', max_iter=10)
>>> harn.on_epoch()
"""
from netharn.metrics import clf_report
dset = harn.datasets[harn.current_tag]
probs = np.vstack(harn._accum_confusion_vectors['probs'])
y_true = np.hstack(harn._accum_confusion_vectors['y_true'])
y_pred = np.hstack(harn._accum_confusion_vectors['y_pred'])
# _pred = probs.argmax(axis=1)
# assert np.all(_pred == y_pred)
# from netharn.metrics import confusion_vectors
# cfsn_vecs = confusion_vectors.ConfusionVectors.from_arrays(
# true=y_true, pred=y_pred, probs=probs, classes=dset.classes)
# report = cfsn_vecs.classification_report()
# combined_report = report['metrics'].loc['combined'].to_dict()
# ovr_cfsn = cfsn_vecs.binarize_ovr()
# Compute multiclass metrics (new way!)
target_names = dset.classes
ovr_report = clf_report.ovr_classification_report(
y_true,
probs,
target_names=target_names,
metrics=['auc', 'ap', 'mcc', 'brier'])
# percent error really isn't a great metric, but its easy and standard.
errors = (y_true != y_pred)
acc = 1.0 - errors.mean()
percent_error = (1.0 - acc) * 100
metrics_dict = ub.odict()
metrics_dict['ave_brier'] = ovr_report['ave']['brier']
metrics_dict['ave_mcc'] = ovr_report['ave']['mcc']
metrics_dict['ave_auc'] = ovr_report['ave']['auc']
metrics_dict['ave_ap'] = ovr_report['ave']['ap']
metrics_dict['percent_error'] = percent_error
metrics_dict['acc'] = acc
harn.info(
ub.color_text('ACC FOR {!r}: {!r}'.format(harn.current_tag, acc),
'yellow'))
# Clear confusion vectors accumulator for the next epoch
harn._accum_confusion_vectors = {
'y_true': [],
'y_pred': [],
'probs': [],
}
return metrics_dict
def color_text(text):
ub.color_text(text)
def main():
def argval(clikey, envkey=None, default=ub.NoParam):
if envkey is not None:
envval = os.environ.get(envkey)
if envval:
default = envval
return ub.argval(clikey, default=default)
DEFAULT_PY_VER = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
PY_VER = argval('--pyver', 'MB_PYTHON_VERSION', default=DEFAULT_PY_VER)
dpath = argval('--dpath', None, default=os.getcwd())
PLAT = argval('--plat', 'PLAT', default='x86_64')
UNICODE_WIDTH = argval('--unicode_width', 'UNICODE_WIDTH', '32')
import multiprocessing
MAKE_CPUS = argval('--make_cpus', 'MAKE_CPUS', multiprocessing.cpu_count() + 1)
OPENCV_VERSION = '4.1.0'
os.chdir(dpath)
BASE = 'manylinux1_{}'.format(PLAT)
BASE_REPO = 'quay.io/skvark'
PY_TAG = 'cp{ver}-cp{ver}m'.format(ver=PY_VER.replace('.', ''))
# do we need the unicode width in this tag?
DOCKER_TAG = '{}-opencv{}-py{}'.format(BASE, OPENCV_VERSION, PY_VER)
if not exists(join(dpath, 'opencv-' + OPENCV_VERSION)):
# FIXME: make robust in the case this fails
fpath = ub.grabdata('https://github.com/opencv/opencv/archive/{}.zip'.format(OPENCV_VERSION), dpath=dpath, verbose=1)
ub.cmd('ln -s {} .'.format(fpath), cwd=dpath, verbose=3)
ub.cmd('unzip {}'.format(fpath), cwd=dpath, verbose=3)
dockerfile_fpath = join(dpath, 'Dockerfile_' + DOCKER_TAG)
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
docker_code = ub.codeblock(
'''
FROM {BASE_REPO}/{BASE}
# SETUP ENV
ARG MB_PYTHON_VERSION={PY_VER}
ENV PYTHON_VERSION={PY_VER}
ENV PYTHON_ROOT=/opt/python/{PY_TAG}/
ENV PYTHONPATH=/opt/python/{PY_TAG}/lib/python{PY_VER}/site-packages/
ENV PATH=/opt/python/{PY_TAG}/bin:$PATH
ENV PYTHON_EXE=/opt/python/{PY_TAG}/bin/python
ENV HOME=/root
ENV PLAT={PLAT}
ENV UNICODE_WIDTH={UNICODE_WIDTH}
# Update python environment
RUN echo "$PYTHON_EXE"
RUN $PYTHON_EXE -m pip install --upgrade pip && \
$PYTHON_EXE -m pip install cmake ninja scikit-build wheel numpy
# This is very different for different operating systems
# https://github.com/skvark/opencv-python/blob/master/setup.py
COPY opencv-{OPENCV_VERSION} /root/code/opencv
RUN mkdir -p /root/code/opencv/build && \
cd /root/code/opencv/build && \
cmake -G "Unix Makefiles" \
-DINSTALL_CREATE_DISTRIB=ON \
-DOPENCV_SKIP_PYTHON_LOADER=ON \
-DBUILD_opencv_apps=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_PERF_TESTS=OFF \
-DBUILD_DOCS=OFF \
-DWITH_QT=OFF \
-DWITH_IPP=OFF \
-DWITH_V4L=ON \
-DBUILD_JPEG=OFF \
-DENABLE_PRECOMPILED_HEADERS=OFF \
/root/code/opencv
# Note: there is no need to compile the above with python
# -DPYTHON3_EXECUTABLE=$PYTHON_EXE \
# -DBUILD_opencv_python3=ON \
# -DOPENCV_PYTHON3_INSTALL_PATH=python \
RUN cd /root/code/opencv/build && make -j{MAKE_CPUS} && make install
'''.format(**locals()))
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code)
docker_build_cli = ' '.join([
'docker', 'build',
# '--build-arg PY_VER={}'.format(PY_VER),
'--tag {}'.format(DOCKER_TAG),
'-f {}'.format(dockerfile_fpath),
'.'
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(info['ret']))
else:
# write out what the tag is
with open(join(dpath, 'opencv-docker-tag.txt'), 'w') as file:
file.write(DOCKER_TAG)
print(ub.color_text('\n--- SUCCESS ---', 'green'))
