def test_global_color_disable():
"""
CommandLine:
xdoctest -m /home/joncrall/code/ubelt/tests/test_color.py test_global_color_disable
"""
import ubelt as ub
text = 'text = "hi"'
has_color = ub.color_text(text, 'red') != ub.color_text(text, None)
text1a = ub.color_text(text, 'red')
text1b = ub.highlight_code(text)
if has_color:
assert text != text1a
assert text != text1b
# Force colors to be disabled
prev = ub.util_colors.NO_COLOR
try:
ub.util_colors.NO_COLOR = True
text2a = ub.color_text(text, 'red')
text2b = ub.highlight_code(text)
assert text == text2a
assert text == text2b
finally:
# Re-enable coloration
ub.util_colors.NO_COLOR = prev
text3a = ub.color_text(text, 'red')
text3b = ub.highlight_code(text)
if has_color:
assert text != text3a
assert text != text3b
def color_pytb(text):
return ub.highlight_code(text, lexer_name='pytb', stripall=True)
def difftext(text1,
text2,
context_lines=0,
ignore_whitespace=False,
colored=False):
r"""
Uses difflib to return a difference string between two similar texts
Args:
text1 (str): old text
text2 (str): new text
context_lines (int): number of lines of unchanged context
ignore_whitespace (bool):
colored (bool): if true highlight the diff
Returns:
str: formatted difference text message
References:
http://www.java2s.com/Code/Python/Utility/IntelligentdiffbetweentextfilesTimPeters.htm
Example:
>>> # build test data
>>> text1 = 'one\ntwo\nthree'
>>> text2 = 'one\ntwo\nfive'
>>> # execute function
>>> result = difftext(text1, text2)
>>> # verify results
>>> print(result)
- three
+ five
Example:
>>> # build test data
>>> text1 = 'one\ntwo\nthree\n3.1\n3.14\n3.1415\npi\n3.4\n3.5\n4'
>>> text2 = 'one\ntwo\nfive\n3.1\n3.14\n3.1415\npi\n3.4\n4'
>>> # execute function
>>> context_lines = 1
>>> result = difftext(text1, text2, context_lines, colored=True)
>>> # verify results
>>> print(result)
"""
import ubelt as ub
import difflib
text1 = ub.ensure_unicode(text1)
text2 = ub.ensure_unicode(text2)
text1_lines = text1.splitlines()
text2_lines = text2.splitlines()
if ignore_whitespace:
text1_lines = [t.rstrip() for t in text1_lines]
text2_lines = [t.rstrip() for t in text2_lines]
ndiff_kw = dict(linejunk=difflib.IS_LINE_JUNK,
charjunk=difflib.IS_CHARACTER_JUNK)
else:
ndiff_kw = {}
all_diff_lines = list(difflib.ndiff(text1_lines, text2_lines, **ndiff_kw))
if context_lines is None:
diff_lines = all_diff_lines
else:
# boolean for every line if it is marked or not
ismarked_list = [
len(line) > 0 and line[0] in '+-?' for line in all_diff_lines
]
# flag lines that are within context_lines away from a diff line
isvalid_list = ismarked_list[:]
for i in range(1, context_lines + 1):
isvalid_list[:-i] = list(
map(any, zip(isvalid_list[:-i], ismarked_list[i:])))
isvalid_list[i:] = list(
map(any, zip(isvalid_list[i:], ismarked_list[:-i])))
USE_BREAK_LINE = True
if USE_BREAK_LINE:
# insert a visual break when there is a break in context
diff_lines = []
prev = False
visual_break = '\n <... FILTERED CONTEXT ...> \n'
#print(isvalid_list)
for line, valid in zip(all_diff_lines, isvalid_list):
if valid:
diff_lines.append(line)
elif prev:
if False:
diff_lines.append(visual_break)
prev = valid
else:
diff_lines = list(ub.compress(all_diff_lines, isvalid_list))
text = '\n'.join(diff_lines)
if colored:
text = ub.highlight_code(text, lexer_name='diff')
return text
def parse(self, string, info=None):
"""
Divide the given string into examples and interleaving text.
Args:
string (str): string representing the doctest
info (dict): info about where the string came from in case of an
error
Returns:
list : a list of `DoctestPart` objects
CommandLine:
python -m xdoctest.parser DoctestParser.parse
Example:
>>> s = 'I am a dummy example with two parts'
>>> x = 10
>>> print(s)
I am a dummy example with two parts
>>> s = 'My purpose it so demonstrate how wants work here'
>>> print('The new want applies ONLY to stdout')
>>> print('given before the last want')
>>> '''
this wont hurt the test at all
even though its multiline '''
>>> y = 20
The new want applies ONLY to stdout
given before the last want
>>> # Parts from previous examples are executed in the same context
>>> print(x + y)
30
this is simply text, and doesnt apply to the previous doctest the
<BLANKLINE> directive is still in effect.
Example:
>>> from xdoctest import parser
>>> from xdoctest.docstr import docscrape_google
>>> from xdoctest import core
>>> self = parser.DoctestParser()
>>> docstr = self.parse.__doc__
>>> blocks = docscrape_google.split_google_docblocks(docstr)
>>> doclineno = self.parse.__func__.__code__.co_firstlineno
>>> key, (string, offset) = blocks[-2]
>>> self._label_docsrc_lines(string)
>>> doctest_parts = self.parse(string)
>>> # each part with a want-string needs to be broken in two
>>> assert len(doctest_parts) == 6
"""
if DEBUG > 1:
print('\n===== PARSE ====')
if sys.version_info.major == 2: # nocover
string = utils.ensure_unicode(string)
if not isinstance(string, six.string_types):
raise TypeError('Expected string but got {!r}'.format(string))
string = string.expandtabs()
# If all lines begin with the same indentation, then strip it.
min_indent = _min_indentation(string)
if min_indent > 0:
string = '\n'.join([l[min_indent:] for l in string.splitlines()])
labeled_lines = None
grouped_lines = None
all_parts = None
try:
labeled_lines = self._label_docsrc_lines(string)
grouped_lines = self._group_labeled_lines(labeled_lines)
all_parts = list(self._package_groups(grouped_lines))
except Exception as orig_ex:
if labeled_lines is None:
failpoint = '_label_docsrc_lines'
elif grouped_lines is None:
failpoint = '_group_labeled_lines'
elif all_parts is None:
failpoint = '_package_groups'
if DEBUG:
print('<FAILPOINT>')
print('!!! FAILED !!!')
print('failpoint = {!r}'.format(failpoint))
import ubelt as ub
import traceback
tb_text = traceback.format_exc()
tb_text = ub.highlight_code(tb_text)
tb_text = ub.indent(tb_text)
print(tb_text)
print('Failed to parse string = <{[<{[<{[')
print(string)
print(']}>a]}>]}> # end string')
print('info = {}'.format(ub.repr2(info)))
print('-----')
print('orig_ex = {}'.format(orig_ex))
print('labeled_lines = {}'.format(ub.repr2(labeled_lines)))
print('grouped_lines = {}'.format(ub.repr2(grouped_lines,
nl=3)))
print('all_parts = {}'.format(ub.repr2(all_parts)))
print('</FAILPOINT>')
# sys.exit(1)
raise exceptions.DoctestParseError(
'Failed to parse doctest in {}'.format(failpoint),
string=string,
info=info,
orig_ex=orig_ex)
if DEBUG > 1:
print('\n===== FINISHED PARSE ====')
return all_parts
def _complete_source(line, state_indent, line_iter):
"""
helper
remove lines from the iterator if they are needed to complete source
"""
norm_line = line[state_indent:] # Normalize line indentation
prefix = norm_line[:4]
suffix = norm_line[4:]
assert prefix.strip() in {'>>>', '...'}, '{}'.format(prefix)
yield line, norm_line
source_parts = [suffix]
# These hacks actually modify the input doctest slighly
HACK_TRIPLE_QUOTE_FIX = True
try:
while not static.is_balanced_statement(source_parts, only_tokens=True):
line_idx, next_line = next(line_iter)
norm_line = next_line[state_indent:]
prefix = norm_line[:4]
suffix = norm_line[4:]
if prefix.strip() not in {'>>>', '...', ''}: # nocover
error = True
if HACK_TRIPLE_QUOTE_FIX:
# TODO: make a more robust patch
if any("'''" in s or '"""' in s for s in source_parts):
# print('HACK FIXING TRIPLE QUOTE')
next_line = next_line[:state_indent] + '... ' + norm_line
norm_line = '... ' + norm_line
prefix = ''
suffix = norm_line
error = False
if error:
if DEBUG:
print(' * !!!ERROR!!!')
print(' * source_parts = {!r}'.format(source_parts))
print(' * prefix = {!r}'.format(prefix))
print(' * norm_line = {!r}'.format(norm_line))
print(' * !!!!!!!!!!!!!')
raise SyntaxError(
'Bad indentation in doctest on line {}: {!r}'.format(
line_idx, next_line))
source_parts.append(suffix)
yield next_line, norm_line
except StopIteration:
if DEBUG:
import ubelt as ub
print('<FAIL DID NOT COMPLETE SOURCE>')
import traceback
tb_text = traceback.format_exc()
tb_text = ub.highlight_code(tb_text)
tb_text = ub.indent(tb_text)
print(tb_text)
# print(' * line_iter = {!r}'.format(line_iter))
print(' * state_indent = {!r}'.format(state_indent))
print(' * line = {!r}'.format(line))
# print('source =\n{}'.format('\n'.join(source_parts)))
print('# Ensure that the following line should actually fail')
print('source_parts = {}'.format(ub.repr2(source_parts, nl=2)))
print(
ub.codeblock(r'''
from xdoctest import static_analysis as static
static.is_balanced_statement(source_parts, only_tokens=False)
static.is_balanced_statement(source_parts, only_tokens=True)
text = '\n'.join(source_parts)
print(text)
static.six_axt_parse(text)
'''))
print('</FAIL DID NOT COMPLETE SOURCE>')
# sys.exit(1)
# TODO: use AST to reparse all doctest parts to discover where the
# syntax error in the doctest is and then raise it.
raise exceptions.IncompleteParseError(
'ill-formed doctest: all parts have been processed '
'but the doctest source is not balanced')
else:
if DEBUG > 1:
import ubelt as ub
print('<SUCCESS COMPLETED SOURCE>')
# print(' * line_iter = {!r}'.format(line_iter))
print('source_parts = {}'.format(ub.repr2(source_parts, nl=2)))
print('</SUCCESS COMPLETED SOURCE>')
def close(closer, visitor):
"""
Populate all undefined names using the context from a module
"""
# Parse the parent module to find only the relevant global varaibles and
# include those in the extracted source code.
closer.debug('closing')
current_sourcecode = closer.current_sourcecode()
# Loop until all undefined names are defined
names = True
while names:
# Determine if there are any variables needed from the parent scope
current_sourcecode = closer.current_sourcecode()
# Make sure we process names in the same order for hashability
prev_names = names
names = sorted(undefined_names(current_sourcecode))
closer.debug(' * undefined_names = {}'.format(names))
if names == prev_names:
print('visitor.definitions = {}'.format(
ub.repr2(visitor.definitions, si=1)))
if DEBUG:
warnings.warn(
'We were unable do do anything about undefined names')
return
else:
current_sourcecode = closer.current_sourcecode()
print('--- <ERROR> ---')
print('Unable to define names')
print(' * names = {!r}'.format(names))
print('<<< CURRENT_SOURCE >>>\n{}\n<<<>>>'.format(
ub.highlight_code(current_sourcecode)))
print('--- </ERROR> ---')
raise AssertionError(
'unable to define names: {}'.format(names))
for name in names:
try:
try:
closer.debug(
' * try visitor.extract_definition({})'.format(
names))
d = visitor.extract_definition(name)
except KeyError as ex:
closer.debug(' * encountered issue: {!r}'.format(ex))
# There is a corner case where we have the definition,
# we just need to move it to the top.
flag = False
for d_ in closer.body_defs.values():
if name == d_.name:
closer.debug(
' * corner case: move definition to top')
closer._add_definition(d_)
flag = True
break
if not flag:
raise
else:
closer.debug(' * add extracted def {}'.format(name))
closer._add_definition(d)
# type_, text = visitor.extract_definition(name)
except Exception as ex:
closer.debug(
' * unable to extracted def {} due to {!r}'.format(
name, ex))
current_sourcecode = closer.current_sourcecode()
print('--- <ERROR> ---')
print('Error computing source code extract_definition')
print(' * failed to close name = {!r}'.format(name))
# print('<<< CURRENT_SOURCE >>>\n{}\n<<<>>>'.format(ub.highlight_code(current_sourcecode)))
print('--- </ERROR> ---')
def main():
def argval(clikey, envkey=None, default=ub.NoParam):
if envkey is not None:
envval = os.environ.get(envkey)
if envval:
default = envval
return ub.argval(clikey, default=default)
DEFAULT_PY_VER = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
PY_VER = argval('--pyver', 'MB_PYTHON_VERSION', default=DEFAULT_PY_VER)
dpath = argval('--dpath', None, default=os.getcwd())
PLAT = argval('--plat', 'PLAT', default='x86_64')
UNICODE_WIDTH = argval('--unicode_width', 'UNICODE_WIDTH', '32')
import multiprocessing
MAKE_CPUS = argval('--make_cpus', 'MAKE_CPUS', multiprocessing.cpu_count() + 1)
OPENCV_VERSION = '4.1.0'
os.chdir(dpath)
BASE = 'manylinux1_{}'.format(PLAT)
BASE_REPO = 'quay.io/skvark'
PY_TAG = 'cp{ver}-cp{ver}m'.format(ver=PY_VER.replace('.', ''))
# do we need the unicode width in this tag?
DOCKER_TAG = '{}-opencv{}-py{}'.format(BASE, OPENCV_VERSION, PY_VER)
if not exists(join(dpath, 'opencv-' + OPENCV_VERSION)):
# FIXME: make robust in the case this fails
fpath = ub.grabdata('https://github.com/opencv/opencv/archive/{}.zip'.format(OPENCV_VERSION), dpath=dpath, verbose=1)
ub.cmd('ln -s {} .'.format(fpath), cwd=dpath, verbose=3)
ub.cmd('unzip {}'.format(fpath), cwd=dpath, verbose=3)
dockerfile_fpath = join(dpath, 'Dockerfile_' + DOCKER_TAG)
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
docker_code = ub.codeblock(
'''
FROM {BASE_REPO}/{BASE}
# SETUP ENV
ARG MB_PYTHON_VERSION={PY_VER}
ENV PYTHON_VERSION={PY_VER}
ENV PYTHON_ROOT=/opt/python/{PY_TAG}/
ENV PYTHONPATH=/opt/python/{PY_TAG}/lib/python{PY_VER}/site-packages/
ENV PATH=/opt/python/{PY_TAG}/bin:$PATH
ENV PYTHON_EXE=/opt/python/{PY_TAG}/bin/python
ENV HOME=/root
ENV PLAT={PLAT}
ENV UNICODE_WIDTH={UNICODE_WIDTH}
# Update python environment
RUN echo "$PYTHON_EXE"
RUN $PYTHON_EXE -m pip install --upgrade pip && \
$PYTHON_EXE -m pip install cmake ninja scikit-build wheel numpy
# This is very different for different operating systems
# https://github.com/skvark/opencv-python/blob/master/setup.py
COPY opencv-{OPENCV_VERSION} /root/code/opencv
RUN mkdir -p /root/code/opencv/build && \
cd /root/code/opencv/build && \
cmake -G "Unix Makefiles" \
-DINSTALL_CREATE_DISTRIB=ON \
-DOPENCV_SKIP_PYTHON_LOADER=ON \
-DBUILD_opencv_apps=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_PERF_TESTS=OFF \
-DBUILD_DOCS=OFF \
-DWITH_QT=OFF \
-DWITH_IPP=OFF \
-DWITH_V4L=ON \
-DBUILD_JPEG=OFF \
-DENABLE_PRECOMPILED_HEADERS=OFF \
/root/code/opencv
# Note: there is no need to compile the above with python
# -DPYTHON3_EXECUTABLE=$PYTHON_EXE \
# -DBUILD_opencv_python3=ON \
# -DOPENCV_PYTHON3_INSTALL_PATH=python \
RUN cd /root/code/opencv/build && make -j{MAKE_CPUS} && make install
'''.format(**locals()))
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code)
docker_build_cli = ' '.join([
'docker', 'build',
# '--build-arg PY_VER={}'.format(PY_VER),
'--tag {}'.format(DOCKER_TAG),
'-f {}'.format(dockerfile_fpath),
'.'
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(info['ret']))
else:
# write out what the tag is
with open(join(dpath, 'opencv-docker-tag.txt'), 'w') as file:
file.write(DOCKER_TAG)
print(ub.color_text('\n--- SUCCESS ---', 'green'))
def main():
# TODO: find a better place for root
ROOT = join(os.getcwd())
# ROOT = '.'
os.chdir(ROOT)
NAME = 'pyhesaff'
VERSION = '0.1.2'
DOCKER_TAG = '{}-{}'.format(NAME, VERSION)
QUAY_REPO = 'quay.io/erotemic/manylinux-for'
DOCKER_URI = '{QUAY_REPO}:{DOCKER_TAG}'.format(**locals())
dockerfile_fpath = join(ROOT, 'Dockerfile')
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
"""
Notes:
docker run --rm -it quay.io/pypa/manylinux2010_x86_64 /bin/bash
---
ls /opt/python
"""
BASE_IMAGE = 'quay.io/pypa/manylinux2010_x86_64'
docker_code = ub.codeblock(f'''
FROM {BASE_IMAGE}
RUN yum install lz4-devel -y
RUN MB_PYTHON_TAG=cp27-cp27m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp27-cp27mu && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp35-cp35m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp36-cp36m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp37-cp37m && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
RUN MB_PYTHON_TAG=cp38-cp38 && \
/opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \
/opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \
source ./venv-$MB_PYTHON_TAG/bin/activate && \
pip install scikit-build cmake ninja
''')
docker_code2 = '\n\n'.join(
[ub.paragraph(p) for p in docker_code.split('\n\n')])
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code2, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code2)
docker_build_cli = ' '.join([
'docker',
'build',
'--tag {}'.format(DOCKER_TAG),
'-f {}'.format(dockerfile_fpath),
'.',
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
if ub.argflag('--dry'):
print('DRY RUN')
print('WOULD RUN')
print(docker_build_cli)
else:
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(
info['ret']))
else:
print(ub.color_text('\n--- SUCCESS ---', 'green'))
print(
ub.highlight_code(
ub.codeblock(r'''
# Finished creating the docker image.
# To test / export / publish you can do something like this:
# Test that we can get a bash terminal
docker run -it {DOCKER_TAG} /bin/bash
# Create a tag for the docker image
docker tag {DOCKER_TAG} {DOCKER_URI}
# Export your docker image to a file
docker save -o ${ROOT}/{DOCKER_TAG}.docker.tar {DOCKER_TAG}
# Login to a docker registry (we are using quay)
# In some cases this works,
docker login
# But you may need to specify secret credentials
load_secrets
echo "QUAY_USERNAME = $QUAY_USERNAME"
docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io
unload_secrets
# Upload the docker image to quay.io
docker push {DOCKER_URI}
''').format(
NAME=NAME,
ROOT=ROOT,
DOCKER_TAG=DOCKER_TAG,
DOCKER_URI=DOCKER_URI,
),
'bash',
))
PUBLISH = 0
if PUBLISH:
cmd1 = 'docker tag {DOCKER_TAG} {DOCKER_URI}'.format(**locals())
cmd2 = 'docker push {DOCKER_URI}'.format(**locals())
print('-- <push cmds> ---')
print(cmd1)
print(cmd2)
print('-- </push cmds> ---')
def main():
import os
ROOT = join(os.getcwd())
ROOT = ub.expandpath('~/code/hesaff')
os.chdir(ROOT)
VERSION = setup.version
PY_VER = sys.version_info.major
NAME = 'pyhesaff'
tag = '{}-{}-py{}'.format(NAME, VERSION, PY_VER)
# context_dpath = ub.ensuredir((ROOT, 'docker/context'))
staging_dpath = ub.ensuredir((ROOT, 'docker/staging'))
# Prestage the multibuild repo
if not exists(join(staging_dpath, 'multibuild')):
# FIXME: make robust in the case this fails
info = ub.cmd(
'git clone https://github.com/matthew-brett/multibuild.git',
cwd=staging_dpath,
verbose=3)
if not exists(join(staging_dpath, 'opencv')):
# FIXME: make robust in the case this fails
opencv_version = '4.1.0'
fpath = ub.grabdata(
'https://github.com/opencv/opencv/archive/{}.zip'.format(
opencv_version),
verbose=1)
ub.cmd('ln -s {} .'.format(fpath), cwd=staging_dpath, verbose=3)
ub.cmd('unzip {}'.format(fpath), cwd=staging_dpath, verbose=3)
import shutil
shutil.move(join(staging_dpath, 'opencv-' + opencv_version),
join(staging_dpath, 'opencv'))
stage_self(ROOT, staging_dpath)
dockerfile_fpath = join(ROOT, 'Dockerfile')
# This docker code is very specific for building linux binaries.
# We will need to do a bit of refactoring to handle OSX and windows.
# But the goal is to get at least one OS working end-to-end.
docker_code = ub.codeblock('''
FROM quay.io/skvark/manylinux1_x86_64
# SETUP ENV
ARG MB_PYTHON_VERSION=3.6
ARG ENABLE_CONTRIB=0
ARG ENABLE_HEADLESS=1
ENV PYTHON_VERSION=3.6
ENV PYTHON_ROOT=/opt/python/cp36-cp36m/
ENV PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages/
ENV PATH=/opt/python/cp36-cp36m/bin:$PATH
ENV PYTHON_EXE=/opt/python/cp36-cp36m/python
ENV MULTIBUILD_DIR=/root/code/multibuild
ENV HOME=/root
# params to bdist_wheel. used to set osx build target.
ENV TEST_DEPENDS="numpy==1.11.1"
ENV BDIST_PARAMS=""
ENV USE_CCACHE=1
ENV PLAT=x86_64
ENV UNICODE_WIDTH=32
# -e BUILD_COMMANDS="$build_cmds" \
# -e PYTHON_VERSION="$MB_PYTHON_VERSION" \
# -e UNICODE_WIDTH="$UNICODE_WIDTH" \
# -e BUILD_COMMIT="$BUILD_COMMIT" \
# -e CONFIG_PATH="$CONFIG_PATH" \
# -e ENV_VARS_PATH="$ENV_VARS_PATH" \
# -e WHEEL_SDIR="$WHEEL_SDIR" \
# -e MANYLINUX_URL="$MANYLINUX_URL" \
# -e BUILD_DEPENDS="$BUILD_DEPENDS" \
# -e USE_CCACHE="$USE_CCACHE" \
# -e REPO_DIR="$repo_dir" \
# -e PLAT="$PLAT" \
# These are defined in the parent image
# ENV JPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include
# ENV JPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a
RUN mkdir -p /io
WORKDIR /root
# Setup code / scripts
COPY docker/staging/multibuild /root/code/multibuild
# Hack to fix issue
RUN find $MULTIBUILD_DIR -iname "*.sh" -type f -exec sed -i 's/gh-clone/gh_clone/g' {} +
# Copy bash configs (mirrors the environs)
COPY docker/config.sh /root/config.sh
COPY docker/bashrc.sh /root/.bashrc
# Setup a virtualenv
RUN source /root/.bashrc && \
$PYTHON_EXE -m pip install --upgrade pip && \
$PYTHON_EXE -m pip install virtualenv && \
$PYTHON_EXE -m virtualenv --python=$PYTHON_EXE $HOME/venv
# Install packages in virtual environment
RUN source /root/.bashrc && \
pip install cmake ninja scikit-build wheel numpy
# This is very different for different operating systems
# https://github.com/skvark/opencv-python/blob/master/setup.py
COPY docker/staging/opencv /root/code/opencv
RUN source /root/.bashrc && \
source code/multibuild/common_utils.sh && \
source code/multibuild/travis_linux_steps.sh && \
mkdir -p /root/code/opencv/build && \
cd /root/code/opencv/build && \
cmake -G "Unix Makefiles" \
-DINSTALL_CREATE_DISTRIB=ON \
-DOPENCV_SKIP_PYTHON_LOADER=ON \
-DBUILD_opencv_apps=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_PERF_TESTS=OFF \
-DBUILD_DOCS=OFF \
-DWITH_QT=OFF \
-DWITH_IPP=OFF \
-DWITH_V4L=ON \
-DBUILD_JPEG=OFF \
-DENABLE_PRECOMPILED_HEADERS=OFF \
-DJPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include \
-DJPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a \
/root/code/opencv
# Note: there is no need to compile the above with python
# -DPYTHON3_EXECUTABLE=$PYTHON_EXE \
# -DBUILD_opencv_python3=ON \
# -DOPENCV_PYTHON3_INSTALL_PATH=python \
RUN source /root/.bashrc && \
source code/multibuild/common_utils.sh && \
source code/multibuild/travis_linux_steps.sh && \
cd /root/code/opencv/build && \
make -j9 && make install
COPY docker/staging/hesaff /root/code/hesaff
# # Use skbuild to build hesaff
# RUN source /root/.bashrc && \
# cd /root/code/hesaff && \
# CMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" python setup.py build_ext --inplace
# export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
# Use cmake to build hesaff 9maybe not needed?)
# RUN source /root/.bashrc && \
# mkdir -p /root/code/hesaff/build && \
# cd /root/code/hesaff/build && \
# CXXFLAGS="-std=c++11 $CXXFLAGS" cmake -G "Unix Makefiles" /root/code/hesaff && \
# make
# Use skbuild to build hesaff
RUN source /root/.bashrc && \
cd /root/code/hesaff && \
python setup.py build && \
python setup.py bdist_wheel
# RUN source /root/.bashrc && \
# pip install xdoctest
''')
try:
print(ub.color_text('\n--- DOCKER CODE ---', 'white'))
print(ub.highlight_code(docker_code, 'docker'))
print(ub.color_text('--- END DOCKER CODE ---\n', 'white'))
except Exception:
pass
with open(dockerfile_fpath, 'w') as file:
file.write(docker_code)
docker_build_cli = ' '.join([
'docker',
'build',
# '--build-arg PY_VER={}'.format(PY_VER),
'--tag {}'.format(tag),
'-f {}'.format(dockerfile_fpath),
'.'
])
print('docker_build_cli = {!r}'.format(docker_build_cli))
info = ub.cmd(docker_build_cli, verbose=3, shell=True)
if info['ret'] != 0:
print(ub.color_text('\n--- FAILURE ---', 'red'))
print('Failed command:')
print(info['command'])
print(info['err'])
print('NOTE: sometimes reruning the command manually works')
raise Exception('Building docker failed with exit code {}'.format(
info['ret']))
else:
print(ub.color_text('\n--- SUCCESS ---', 'green'))
DEPLOY = True
if DEPLOY:
VMNT_DIR = '{ROOT}/{NAME}-docker/vmnt'.format(NAME=NAME, ROOT=ROOT)
print('VMNT_DIR = {!r}'.format(VMNT_DIR))
ub.ensuredir(VMNT_DIR)
# TODO: Correctly mangle the ffmpeg libs as done via
# ls -a ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/
# ls ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/.libs
# cp code/hesaff/build/libhesaff.so /root/vmnt
# cp /root/ffmpeg_build/lib/libavcodec.so.58 /root/vmnt
# cp /root/ffmpeg_build/lib/libavformat.so.58 /root/vmnt
# cp /root/ffmpeg_build/lib/libavutil.so.56 /root/vmnt
# cp /root/ffmpeg_build/lib/libswscale.so.5 /root/vmnt
inside_cmds = ' && '.join(
ub.codeblock('''
cp code/hesaff/dist/pyhesaff*.whl /root/vmnt
''').split('\n'))
docker_run_cli = ' '.join([
'docker', 'run', '-v {}:/root/vmnt/'.format(VMNT_DIR), '-it', tag,
'bash -c "{}"'.format(inside_cmds)
])
print(docker_run_cli)
info = ub.cmd(docker_run_cli, verbose=3)
assert info['ret'] == 0
# import shutil
# PKG_DIR = join(ROOT, 'pyhesaff')
# shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libhesaff-manylinux1_x86_64.so'))
# TODO: do this correctly
# shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libavcodec.so.58'))
# shutil.copy(join(VMNT_DIR, 'libavformat.so.58'), join(PKG_DIR, 'libavformat.so.58'))
# shutil.copy(join(VMNT_DIR, 'libavutil.so.56'), join(PKG_DIR, 'libavutil.so.56'))
# shutil.copy(join(VMNT_DIR, 'libswscale.so.5'), join(PKG_DIR, 'libswscale.so.5'))
# print(ub.highlight_code(ub.codeblock(
print(
ub.highlight_code(
ub.codeblock(r'''
# Finished creating the docker image.
# To test / export you can do something like this:
VMNT_DIR={ROOT}/{NAME}-docker/vmnt
mkdir -p VMNT_DIR
TAG={tag}
# Test that we can get a bash terminal
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash
# Move deployment to the vmnt directory
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 -m xdoctest pyhesaff'
# Run system tests
docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 run_doctests.sh'
# Inside bash test that we can fit a new model
python -m pyhessaff demo
mkdir -p ${ROOT}/{NAME}-docker/dist
docker save -o ${ROOT}/{NAME}-docker/dist/{tag}.docker.tar {tag}
''').format(NAME=NAME, ROOT=ROOT, tag=tag), 'bash'))
def color_pytb(text):
return ub.highlight_code(text, lexer_name='pytb', stripall=True)