def prefixes_should_apply_to_run(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def prefixes_should_apply_to_run(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def prefixes_should_apply_to_sudo(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
ctx.sudo('whoami')
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
ok_(runner.run.called, "sudo() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def prefixes_should_apply_to_run(self, Local):
runner = Local.return_value
c = Context()
with c.prefix("cd foo"):
c.run("whoami")
cmd = "cd foo && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def prefixes_should_apply_to_sudo(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
ctx.sudo('whoami')
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
ok_(runner.run.called, "sudo() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def prefixes_should_apply_to_sudo(self, Local):
runner = Local.return_value
c = Context()
with c.prefix("cd foo"):
c.sudo("whoami")
cmd = "sudo -S -p '[sudo] password: ' cd foo && whoami"
assert runner.run.called, "sudo() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('source venv'):
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && source venv && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
c = Context()
with c.prefix("source venv"):
with c.cd("foo"):
c.run("whoami")
cmd = "cd foo && source venv && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def cd_should_occur_before_prefixes(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('source venv'):
with ctx.cd('foo'):
ctx.run('whoami')
cmd = "cd foo && source venv && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def nesting_should_retain_order(self, Local):
runner = Local.return_value
c = Context()
with c.prefix('cd foo'):
with c.prefix('cd bar'):
c.run('whoami')
cmd = "cd foo && cd bar && whoami"
assert runner.run.called, "run() never called runner.run()!" # noqa
assert runner.run.call_args[0][0] == cmd
c.run('whoami')
cmd = "cd foo && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
# also test that prefixes do not persist
c.run('whoami')
cmd = "whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def nesting_should_retain_order(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
with ctx.prefix('cd bar'):
ctx.run('whoami')
cmd = "cd foo && cd bar && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
# also test that prefixes do not persist
ctx.run('whoami')
cmd = "whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def nesting_should_retain_order(self, Local):
runner = Local.return_value
ctx = Context()
with ctx.prefix('cd foo'):
with ctx.prefix('cd bar'):
ctx.run('whoami')
cmd = "cd foo && cd bar && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
ctx.run('whoami')
cmd = "cd foo && whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
# also test that prefixes do not persist
ctx.run('whoami')
cmd = "whoami"
ok_(runner.run.called, "run() never called runner.run()!")
eq_(runner.run.call_args[0][0], cmd)
def nesting_should_retain_order(self, Local):
runner = Local.return_value
c = Context()
with c.prefix('cd foo'):
with c.prefix('cd bar'):
c.run('whoami')
cmd = "cd foo && cd bar && whoami"
assert runner.run.called, "run() never called runner.run()!" # noqa
assert runner.run.call_args[0][0] == cmd
c.run('whoami')
cmd = "cd foo && whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
# also test that prefixes do not persist
c.run('whoami')
cmd = "whoami"
assert runner.run.called, "run() never called runner.run()!"
assert runner.run.call_args[0][0] == cmd
def _run_task(
ctx: Context,
connector_string: str,
task_name: str,
multi_envs: bool = True,
module_path: Optional[str] = None,
task_commands: Dict = TASK_COMMANDS,
**kwargs: Any,
) -> int:
"""
Run task in its own environment.
"""
cur_dir = os.getcwd()
if multi_envs:
if module_path:
os.chdir(module_path)
source_path = connector_string
else:
os.chdir(os.path.join(CONNECTORS_DIR,
f"source-{connector_string}"))
source_path = f"source_{connector_string.replace('-', '_')}"
else:
source_path = connector_string
venv_name = tempfile.mkdtemp(dir=os.curdir)
virtualenv.cli_run([venv_name])
activator = os.path.join(os.path.abspath(venv_name), "bin", "activate")
commands = []
commands.extend([
cmd.format(source_path=source_path, venv=venv_name, **kwargs)
for cmd in task_commands[task_name]
])
exit_code: int = 0
try:
with ctx.prefix(f"source {activator}"):
for command in commands:
result = ctx.run(command, echo=True, warn=True)
if result.return_code:
exit_code = 1
break
finally:
shutil.rmtree(venv_name, ignore_errors=True)
if module_path:
os.chdir(cur_dir)
return exit_code
def should_use_finally_to_revert_changes_on_exceptions(self, Local):
class Oops(Exception):
pass
runner = Local.return_value
c = Context()
try:
with c.prefix("cd foo"):
c.run("whoami")
assert runner.run.call_args[0][0] == "cd foo && whoami"
raise Oops
except Oops:
pass
c.run("ls")
# When bug present, this would be "cd foo && ls"
assert runner.run.call_args[0][0] == "ls"
def docs(ctx: Context, html: Boolean = True, pdf: Boolean = True):
"""Build the documentation.
Parameters
----------
ctx
Context.
html
Whether to build the *HTML* documentation.
pdf
Whether to build the *PDF* documentation.
"""
with ctx.prefix("export COLOUR_SCIENCE__DOCUMENTATION_BUILD=True"):
with ctx.cd("docs"):
if html:
message_box('Building "HTML" documentation...')
ctx.run("make html")
if pdf:
message_box('Building "PDF" documentation...')
ctx.run("make latexpdf")
def install_requirements_pip(ctx: Context):
"""Install requirements in the virtual environment."""
# Collect all parameters determining installation of requirements
pip_reqs = set([])
req_fns = set([])
for _conda_req, pip_req in ctx.project.requirements:
if pip_req is not None:
pip_reqs.add(pip_req)
for package in ctx.project.packages:
for toolname in package.tools:
tool = TOOLS[toolname]
for _conda_req, pip_req in tool.requirements:
if pip_req is not None:
pip_reqs.add(pip_req)
req_fns.add(os.path.join(package.path, "setup.py"))
req_hash = compute_req_hash(pip_reqs, req_fns)
fn_skip = os.path.join(ctx.testenv.path, ".skip_install")
if check_install_requirements(fn_skip, req_hash):
with ctx.prefix(ctx.testenv.activate):
if len(pip_reqs) > 0:
# Install pip packages for the tools
ctx.run("pip install -U {}".format(" ".join(
"'{}'".format(pip_req) for pip_req in pip_reqs)))
# Install dependencies for the project.
for package in ctx.project.packages:
with ctx.cd(package.path):
ctx.run("python setup.py egg_info")
fn_requires = os.path.join(
package.dist_name.replace("-", "_") + ".egg-info",
"requires.txt")
with tempfile.TemporaryDirectory() as tmpdir:
fn_requirements = os.path.join(tmpdir, "requirements.txt")
convert_requires(fn_requires, fn_requirements)
ctx.run("pip install -U -r " + fn_requirements)
# Update the timestamp on the skip file.
with open(fn_skip, 'w') as f:
f.write(req_hash + '\n')
def install_requirements_conda(ctx: Context):
"""Install all requirements, including tools used by Roberto."""
# Collect all parameters determining the install commands (to good
# approximation) and turn them into a hash.
# Some conda requirements are included by default because they must be present:
# - conda: to make sure it is always up to date.
# - conda-build: to have conda-render for getting requirements from recipes.
conda_reqs = set(["conda"])
pip_reqs = set([])
recipe_dirs = []
# Add project as a tool because it also contains requirements.
tools = [ctx.project]
for package in ctx.project.packages:
for toolname in package.tools:
tools.append(ctx.tools[toolname])
recipe_dir = os.path.join(package.path, "tools", "conda.recipe")
if os.path.isdir(recipe_dir):
recipe_dirs.append(recipe_dir)
else:
print("Skipping recipe {}. (directory does not exist)".format(recipe_dir))
for tool in tools:
for conda_req, pip_req in tool.get("requirements", []):
if conda_req is None:
pip_reqs.add(pip_req)
conda_reqs.add("pip")
else:
conda_reqs.add(conda_req)
req_hash = compute_req_hash(
set("conda:" + conda_req for conda_req in conda_reqs) |
set("pip:" + pip_req for pip_req in pip_reqs),
sum([glob(os.path.join(recipe_dir, "*")) for recipe_dir in recipe_dirs], [])
)
fn_skip = os.path.join(ctx.testenv.path, ".skip_install")
if check_install_requirements(fn_skip, req_hash):
with ctx.prefix(ctx.conda.activate_base):
# Update conda packages in the base env. Conda packages in the dev env
# tend to be ignored.
ctx.run("conda install --update-deps -y {}".format(
" ".join("'{}'".format(conda_req) for conda_req
in conda_reqs if conda_req.startswith('conda'))))
with ctx.prefix(ctx.testenv.activate):
# Update packages already installed
ctx.run("conda update --all -y")
# Update and install other requirements for Roberto, in the dev env.
ctx.run("conda install --update-deps -y {}".format(" ".join(
"'{}'".format(conda_req) for conda_req in conda_reqs
if not conda_req.startswith('conda'))))
print("Rendering conda package, extracting requirements, which will be installed.")
# Install dependencies from recipes, excluding own packages.
own_conda_reqs = [package.dist_name for package in ctx.project.packages]
for recipe_dir in recipe_dirs:
# Send the output of conda render to a temporary directory.
with tempfile.TemporaryDirectory() as tmpdir:
rendered_path = os.path.join(tmpdir, "rendered.yml")
ctx.run(
"conda render -f {} {} --variants {}".format(
rendered_path, recipe_dir, ctx.conda.variants))
with open(rendered_path) as f:
rendered = yaml.safe_load(f)
# Build a (simplified) list of requirements and install.
dep_conda_reqs = set([])
req_sources = [
("requirements", 'build'),
("requirements", 'host'),
("requirements", 'run'),
("test", 'requires'),
]
for req_section, req_type in req_sources:
for recipe_req in rendered.get(req_section, {}).get(req_type, []):
words = recipe_req.split()
if words[0] not in own_conda_reqs:
dep_conda_reqs.add(" ".join(words[:2]))
ctx.run("conda install --update-deps -y {}".format(" ".join(
"'{}'".format(conda_req) for conda_req in dep_conda_reqs)))
# Update and install requirements for Roberto from pip, if any.
if pip_reqs:
ctx.run("pip install --upgrade {}".format(" ".join(
"'{}'".format(pip_req) for pip_req in pip_reqs)))
# Update the timestamp on the skip file.
with open(fn_skip, 'w') as f:
f.write(req_hash + '\n')
