def build(self, ctx: Context) -> None:
if self.platform in [
SupportedPlatformEnum.WINDOWS_32,
SupportedPlatformEnum.WINDOWS_64
]:
if self.platform == SupportedPlatformEnum.WINDOWS_32:
arch = "x86"
build_script = "bld_ml32.bat"
build_platform = "Win32"
else:
arch = "x64"
build_script = "bld_ml64.bat"
build_platform = "x64"
masm_path = self.src_path / "contrib" / f"masm{arch}"
with ctx.cd(str(masm_path)):
ctx.run(build_script)
ctx.run(
f"msbuild ..\\vstudio\\vc14\\zlibvc.sln /P:Configuration=Release /P:Platform={build_platform}"
)
else:
# Linux/macOS build
with ctx.cd(str(self.src_path)):
ctx.run('CFLAGS="-fPIC" ./configure -static')
ctx.run("make clean")
ctx.run("make")
def build(self, ctx: Context) -> None:
if self.platform in [
SupportedPlatformEnum.WINDOWS_32,
SupportedPlatformEnum.WINDOWS_64
]:
if self.platform == SupportedPlatformEnum.WINDOWS_32:
arch = 'x86'
build_script = 'bld_ml32.bat'
build_platform = 'Win32'
else:
arch = 'x64'
build_script = 'bld_ml64.bat'
build_platform = 'x64'
masm_path = self.src_path / 'contrib' / f'masm{arch}'
with ctx.cd(str(masm_path)):
ctx.run(build_script)
ctx.run(
f'msbuild ..\\vstudio\\vc14\\zlibvc.sln /P:Configuration=Release /P:Platform={build_platform}'
)
else:
# Linux/macOS build
with ctx.cd(str(self.src_path)):
ctx.run('CFLAGS="-fPIC" ./configure -static')
ctx.run('make clean')
ctx.run('make')
def virtualise(ctx: Context, tests: Boolean = True):
"""Create a virtual environment for the project build.
Parameters
----------
ctx
Context.
tests
Whether to run tests on the virtual environment.
"""
unique_name = f"{PYPI_PACKAGE_NAME}-{uuid.uuid1()}"
with ctx.cd("dist"):
ctx.run(f"tar -xvf {PYPI_PACKAGE_NAME}-{APPLICATION_VERSION}.tar.gz")
ctx.run(f"mv {PYPI_PACKAGE_NAME}-{APPLICATION_VERSION} {unique_name}")
with ctx.cd(unique_name):
ctx.run('poetry install --extras "plotting"')
ctx.run("source $(poetry env info -p)/bin/activate")
ctx.run('python -c "import imageio;'
'imageio.plugins.freeimage.download()"')
if tests:
# The test images are not available thus many doctests
# cannot be run properly:
# "--doctest-modules "
ctx.run(
"poetry run py.test "
"--disable-warnings "
f"--ignore={PYTHON_PACKAGE_NAME}/examples "
f"{PYTHON_PACKAGE_NAME}",
env={"MPLBACKEND": "AGG"},
)
def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None:
if local:
with c.cd(DAEMON_DIR):
c.run("poetry build -f wheel", hide=hide)
c.run("sudo python3 -m pip install dist/*")
else:
args = "" if dev else "--no-dev"
with c.cd(DAEMON_DIR):
c.run(f"poetry install {args}", hide=hide)
if dev:
c.run("poetry run pre-commit install", hide=hide)
def install_poetry(c: Context, dev: bool, hide: bool) -> None:
c.run("pipx install poetry", hide=hide)
args = "" if dev else "--no-dev"
with c.cd(DAEMON_DIR):
c.run(f"poetry install {args}", hide=hide)
if dev:
c.run("poetry run pre-commit install", hide=hide)
def get_python(c: Context, warn: bool = False) -> str:
with c.cd(DAEMON_DIR):
r = c.run("poetry env info -p", warn=warn, hide=True)
if r.ok:
venv = r.stdout.strip()
return os.path.join(venv, "bin", "python")
else:
return ""
def cd_should_apply_to_sudo(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.cd('foo'):
ctx.sudo('whoami')
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
ok_(runner.run.called, "sudo() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_apply_to_run(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_apply_to_run(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_apply_to_sudo(self, Local):
runner = Local.return_value
c = Context()
with c.cd("foo"):
c.sudo("whoami")
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
assert runner.run.called, "sudo() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def cd_should_apply_to_run(self, Local):
runner = Local.return_value
c = Context()
with c.cd("foo"):
c.run("whoami")
cmd = "cd foo && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def cd_should_apply_to_sudo(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.cd('foo'):
ctx.sudo('whoami')
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
ok_(runner.run.called, "sudo() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
c = Context()
with c.prefix("source venv"):
with c.cd("foo"):
c.run("whoami")
cmd = "cd foo && source venv && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('source venv'):
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && source venv && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('source venv'):
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && source venv && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def sha256(ctx: Context):
"""Compute the project *Pypi* package *sha256* with *OpenSSL*.
Parameters
----------
ctx
Context.
"""
message_box('Computing "sha256"...')
with ctx.cd("dist"):
ctx.run(f"openssl sha256 {PYPI_PACKAGE_NAME}-*.tar.gz")
def release(ctx: Context):
"""Release the project to *Pypi* with *Twine*.
Parameters
----------
ctx
Context.
"""
message_box("Releasing...")
with ctx.cd("dist"):
ctx.run("twine upload *.tar.gz")
ctx.run("twine upload *.whl")
def _build():
"""
Build local support docs tree and return the build target dir for cleanup.
"""
c = Context()
support = join(dirname(__file__), "_support")
docs = join(support, "docs")
build = join(support, "_build")
command = "sphinx-build -c {} -W {} {}".format(support, docs, build)
with c.cd(support):
# Turn off stdin mirroring to avoid irritating pytest.
c.run(command, in_stream=False)
return build
def todo(ctx: Context):
"""Export the TODO items.
Parameters
----------
ctx
Context.
"""
message_box('Exporting "TODO" items...')
with ctx.cd("utilities"):
ctx.run("./export_todo.py")
def build(self, ctx: Context) -> None:
if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
if self.platform == SupportedPlatformEnum.WINDOWS_32:
arch = 'x86'
build_script = 'bld_ml32.bat'
build_platform = 'Win32'
else:
arch = 'x64'
build_script = 'bld_ml64.bat'
build_platform = 'x64'
masm_path = self.src_path / 'contrib' / f'masm{arch}'
with ctx.cd(str(masm_path)):
ctx.run(build_script)
ctx.run(f'msbuild ..\\vstudio\\vc14\\zlibvc.sln /P:Configuration=Release /P:Platform={build_platform}')
else:
# Linux/macOS build
with ctx.cd(str(self.src_path)):
ctx.run('CFLAGS="-fPIC" ./configure -static')
ctx.run('make clean')
ctx.run('make')
def test_full_integration(folder, cmd, expected_output, py2_only, tmpdir):
# --durations=10, you will see each one gets run twice, maybe fix?
ctx = Context()
assert ctx.pformat()
with ctx.cd("sites/magic_docs/examples/{}".format(folder)):
result = ctx.run("TMPDIR={} {}".format(tmpdir, cmd),
hide=True,
warn=py2_only).stdout
try:
assert expected_output in result
except:
if py2_only and six.PY2:
pytest.xfail("We knew that.")
raise
def build(self,
ctx: Context,
zlib_lib_path: Optional[Path] = None,
zlib_include_path: Optional[Path] = None,
should_build_for_debug: bool = False) -> None:
if not zlib_lib_path or not zlib_include_path:
raise ValueError('Missing argument')
# Build OpenSSL
openssl_target = self._get_build_target(should_build_for_debug)
with ctx.cd(str(self.src_path)):
self._run_configure_command(ctx, openssl_target, zlib_lib_path,
zlib_include_path)
self._run_build_steps(ctx)
def build(
self,
ctx: Context,
zlib_lib_path: Optional[Path] = None,
zlib_include_path: Optional[Path] = None,
should_build_for_debug: bool = False
) -> None:
if not zlib_lib_path or not zlib_include_path:
raise ValueError('Missing argument')
# Build OpenSSL
openssl_target = self._get_build_target(should_build_for_debug)
with ctx.cd(str(self.src_path)):
self._run_configure_command(ctx, openssl_target, zlib_lib_path, zlib_include_path)
self._run_build_steps(ctx)
def exec_in_dir(base_command, directory, options='', tee=None, exit_on_failure=True):
if tee:
tee = sum_paths(directory, tee)
# NOTE: Path doesn't seem to be compatible with cd here,
# see PR https://j.mp/3dq3nvf
directory = str(directory)
config = Config()
config.load_shell_env()
context = Context(config=config)
# with context.cd(str(directory)):
with context.cd(directory):
exec(base_command, options, exit_on_failure, tee=tee, context=context)
def cd_should_accept_any_stringable_object(self, Local):
class Path(object):
def __init__(self, value):
self.value = value
def __str__(self):
return self.value
runner = Local.return_value
c = Context()
with c.cd(Path("foo")):
c.run("whoami")
cmd = "cd foo && whoami"
assert runner.run.call_args[0][0] == cmd
def should_use_finally_to_revert_changes_on_exceptions(self, Local):
class Oops(Exception):
pass
runner = Local.return_value
c = Context()
try:
with c.cd("foo"):
c.run("whoami")
assert runner.run.call_args[0][0] == "cd foo && whoami"
raise Oops
except Oops:
pass
c.run("ls")
# When bug present, this would be "cd foo && ls"
assert runner.run.call_args[0][0] == "ls"
def formatting(
ctx: Context,
asciify: Boolean = True,
bibtex: Boolean = True,
):
"""Convert unicode characters to ASCII and cleanup the *BibTeX* file.
Parameters
----------
ctx
Context.
asciify
Whether to convert unicode characters to ASCII.
bibtex
Whether to cleanup the *BibTeX* file.
"""
if asciify:
message_box("Converting unicode characters to ASCII...")
with ctx.cd("utilities"):
ctx.run("./unicode_to_ascii.py")
if bibtex:
message_box('Cleaning up "BibTeX" file...')
bibtex_path = BIBLIOGRAPHY_NAME
with open(bibtex_path) as bibtex_file:
entries = (biblib.bib.Parser().parse(
bibtex_file.read()).get_entries())
for entry in sorted(entries.values(), key=lambda x: x.key):
try:
del entry["file"]
except KeyError:
pass
for key, value in entry.items():
entry[key] = re.sub("(?<!\\\\)\\&", "\\&", value)
with open(bibtex_path, "w") as bibtex_file:
for entry in sorted(entries.values(), key=lambda x: x.key):
bibtex_file.write(entry.to_bib())
bibtex_file.write("\n")
def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None:
if c.run("sudo which zebra", warn=True, hide=hide):
print("\nquagga already installed, skipping ospf mdr")
return
if os_info.like == OsLike.DEBIAN:
c.run("sudo apt install -y libtool gawk libreadline-dev git",
hide=hide)
elif os_info.like == OsLike.REDHAT:
c.run("sudo yum install -y libtool gawk readline-devel git", hide=hide)
ospf_dir = "../ospf-mdr"
ospf_url = "https://github.com/USNavalResearchLaboratory/ospf-mdr.git"
c.run(f"git clone {ospf_url} {ospf_dir}", hide=hide)
with c.cd(ospf_dir):
c.run(f"git checkout {OSPFMDR_CHECKOUT}", hide=hide)
c.run("./bootstrap.sh", hide=hide)
c.run(
"./configure --disable-doc --enable-user=root --enable-group=root "
"--with-cflags=-ggdb --sysconfdir=/usr/local/etc/quagga --enable-vtysh "
"--localstatedir=/var/run/quagga",
hide=hide)
c.run("make -j$(nproc)", hide=hide)
c.run("sudo make install", hide=hide)
def docs(ctx: Context, html: Boolean = True, pdf: Boolean = True):
"""Build the documentation.
Parameters
----------
ctx
Context.
html
Whether to build the *HTML* documentation.
pdf
Whether to build the *PDF* documentation.
"""
with ctx.prefix("export COLOUR_SCIENCE__DOCUMENTATION_BUILD=True"):
with ctx.cd("docs"):
if html:
message_box('Building "HTML" documentation...')
ctx.run("make html")
if pdf:
message_box('Building "PDF" documentation...')
ctx.run("make latexpdf")
def install_requirements_pip(ctx: Context):
"""Install requirements in the virtual environment."""
# Collect all parameters determining installation of requirements
pip_reqs = set([])
req_fns = set([])
for _conda_req, pip_req in ctx.project.requirements:
if pip_req is not None:
pip_reqs.add(pip_req)
for package in ctx.project.packages:
for toolname in package.tools:
tool = TOOLS[toolname]
for _conda_req, pip_req in tool.requirements:
if pip_req is not None:
pip_reqs.add(pip_req)
req_fns.add(os.path.join(package.path, "setup.py"))
req_hash = compute_req_hash(pip_reqs, req_fns)
fn_skip = os.path.join(ctx.testenv.path, ".skip_install")
if check_install_requirements(fn_skip, req_hash):
with ctx.prefix(ctx.testenv.activate):
if len(pip_reqs) > 0:
# Install pip packages for the tools
ctx.run("pip install -U {}".format(" ".join(
"'{}'".format(pip_req) for pip_req in pip_reqs)))
# Install dependencies for the project.
for package in ctx.project.packages:
with ctx.cd(package.path):
ctx.run("python setup.py egg_info")
fn_requires = os.path.join(
package.dist_name.replace("-", "_") + ".egg-info",
"requires.txt")
with tempfile.TemporaryDirectory() as tmpdir:
fn_requirements = os.path.join(tmpdir, "requirements.txt")
convert_requires(fn_requires, fn_requirements)
ctx.run("pip install -U -r " + fn_requirements)
# Update the timestamp on the skip file.
with open(fn_skip, 'w') as f:
f.write(req_hash + '\n')
def apollo_query_typedefs(context: Context, watch: bool = False):
# well. the working directory cannot be set to 'apollo codegen:generate' so we need to cd to it (globalTypes.ts must be inside the
# assets/ts folder to be importable)
from pathlib import Path
import shutil
for path in Path().rglob('__generated__'):
shutil.rmtree(path)
print("__generated__ directories deleted")
command = [
"npx apollo codegen:generate",
"--target typescript",
f"--localSchemaFile=schema.graphql",
"--includes=./**/*.ts*",
"--tagName=gql",
]
if watch:
command.append('--watch')
with context.cd(Folders.frontend.__str__()):
context.run(' '.join(command))
def runserver_internal(c: Context):
with c.cd(site_name):
c.run(sys.executable + " manage.py runserver " + str(port))
def build(ctx: Context):
"""Build the project and runs dependency tasks, i.e., *docs*, *todo*, and
*preflight*.
Parameters
----------
ctx
Context.
"""
message_box("Building...")
ctx.run("poetry build")
with ctx.cd("dist"):
ctx.run(f"tar -xvf {PYPI_PACKAGE_NAME}-{APPLICATION_VERSION}.tar.gz")
ctx.run(f"cp {PYPI_PACKAGE_NAME}-{APPLICATION_VERSION}/setup.py ../")
ctx.run(f"rm -rf {PYPI_PACKAGE_NAME}-{APPLICATION_VERSION}")
with open("setup.py") as setup_file:
source = setup_file.read()
setup_kwargs = []
def sub_callable(match):
setup_kwargs.append(match)
return ""
template = """
setup({0}
)
"""
source = re.sub(
"from setuptools import setup",
('"""\n'
"Colour - Checker Detection - Setup\n"
"==================================\n"
'"""\n\n'
"import codecs\n"
"from setuptools import setup"),
source,
)
source = re.sub(
"setup_kwargs = {(.*)}.*setup\\(\\*\\*setup_kwargs\\)",
sub_callable,
source,
flags=re.DOTALL,
)[:-2]
setup_kwargs = setup_kwargs[0].group(1).splitlines()
for i, line in enumerate(setup_kwargs):
setup_kwargs[i] = re.sub("^\\s*('(\\w+)':\\s?)", " \\2=", line)
if setup_kwargs[i].strip().startswith("long_description"):
setup_kwargs[i] = (" long_description="
"codecs.open('README.rst', encoding='utf8')"
".read(),")
source += template.format("\n".join(setup_kwargs))
with open("setup.py", "w") as setup_file:
setup_file.write(source)
ctx.run("poetry run pre-commit run --files setup.py || true")
ctx.run("twine check dist/*")
def get_pytest(c: Context) -> str:
with c.cd(DAEMON_DIR):
venv = c.run("poetry env info -p", hide=True).stdout.strip()
return os.path.join(venv, "bin", "pytest")
