def test_build_dependencies_virtualenv_multiple(tmp_path):
"""A virtualenv is created with multiple requirements files."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=["reqs1.txt", "reqs2.txt"],
)
with patch("charmcraft.charm_builder.subprocess.run") as mock_run:
mock_run.return_value.returncode = 1
with patch("charmcraft.charm_builder._process_run") as mock:
mock.return_value = 0
builder.handle_dependencies()
envpath = build_dir / VENV_DIRNAME
assert mock.mock_calls == [
call(["pip3", "--version"]),
call([
"pip3",
"install",
"--target={}".format(envpath),
"--requirement=reqs1.txt",
"--requirement=reqs2.txt",
]),
]
def test_build_dependencies_virtualenv_binary_packages(tmp_path):
"""A virtualenv is created with the specified packages."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=["pkg1", "pkg2"],
python_packages=[],
requirements=[],
)
with patch("charmcraft.charm_builder._process_run") as mock:
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
pip_cmd = str(charm_builder._find_venv_bin(tmp_path / STAGING_VENV_DIRNAME, "pip3"))
assert mock.mock_calls == [
call(["python3", "-m", "venv", str(tmp_path / STAGING_VENV_DIRNAME)]),
call([pip_cmd, "--version"]),
call([pip_cmd, "install", "--upgrade", "pkg1", "pkg2"]),
]
site_packages_dir = charm_builder._find_venv_site_packages(pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [call(site_packages_dir, build_dir / VENV_DIRNAME)]
def test_build_dependencies_needs_system(tmp_path, config):
"""pip3 is called with --system when pip3 needs it."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=["reqs"],
)
with patch("charmcraft.charm_builder.subprocess.run") as mock_run:
mock_run.return_value.returncode = 0
with patch("charmcraft.charm_builder._process_run") as mock:
mock.return_value = 0
builder.handle_dependencies()
envpath = build_dir / VENV_DIRNAME
assert mock.mock_calls == [
call(["pip3", "--version"]),
call([
"pip3",
"install",
"--target={}".format(envpath),
"--system",
"--requirement=reqs",
]),
]
def test_build_dependencies_no_reused_missing_hash_file(tmp_path, emitter):
"""Dependencies are built again because previous hash file was not found."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=[],
python_packages=["ops"],
requirements=[],
)
staging_venv_dir = tmp_path / STAGING_VENV_DIRNAME
# patch the dependencies installation method so it skips all subprocessing but actually
# creates the directory, to simplify testing
builder._install_dependencies = lambda dirpath: dirpath.mkdir(exist_ok=True
)
# first run!
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("Dependencies directory not found")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed
assert staging_venv_dir.exists()
# installation directory copied to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
# remove the hash file
(tmp_path / DEPENDENCIES_HASH_FILENAME).unlink()
# second run!
emitter.interactions.clear()
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("Dependencies hash file not found")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed *again*
assert staging_venv_dir.exists()
# installation directory copied *again* to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
def test_build_dependencies_reused(tmp_path, emitter):
"""Happy case to reuse dependencies from last run."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
reqs_file = tmp_path / "reqs.txt"
reqs_file.touch()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=[],
python_packages=[],
requirements=[reqs_file],
)
staging_venv_dir = tmp_path / STAGING_VENV_DIRNAME
# patch the dependencies installation method so it skips all subprocessing but actually
# creates the directory, to simplify testing; note that we specifically are calling mkdir
# to fail if the directory is already there, so we ensure it is called once
builder._install_dependencies = lambda dirpath: dirpath.mkdir(exist_ok=
False)
# first run!
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("Dependencies directory not found")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed
assert staging_venv_dir.exists()
# installation directory copied to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
# second run!
emitter.interactions.clear()
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace(
"Reusing installed dependencies, they are equal to last run ones")
# installation directory copied *again* to the build directory (this is always done as
# buildpath is cleaned)
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
def test_build_dependencies_virtualenv_all(tmp_path, emitter):
"""A virtualenv is created with the specified packages."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
reqs_file_1 = tmp_path / "reqs.txt"
reqs_file_1.touch()
reqs_file_2 = tmp_path / "reqs.txt"
reqs_file_1.touch()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=["pkg1", "pkg2"],
python_packages=["pkg3", "pkg4"],
requirements=[reqs_file_1, reqs_file_2],
)
with patch("charmcraft.charm_builder._process_run") as mock:
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
pip_cmd = str(
charm_builder._find_venv_bin(tmp_path / STAGING_VENV_DIRNAME, "pip3"))
assert mock.mock_calls == [
call(["python3", "-m", "venv",
str(tmp_path / STAGING_VENV_DIRNAME)]),
call([pip_cmd, "--version"]),
call([pip_cmd, "install", "--upgrade", "pkg1", "pkg2"]),
call([
pip_cmd, "install", "--upgrade", "--no-binary", ":all:", "pkg3",
"pkg4"
]),
call([
pip_cmd,
"install",
"--upgrade",
"--no-binary",
":all:",
f"--requirement={reqs_file_1}",
f"--requirement={reqs_file_2}",
]),
]
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
emitter.assert_trace("Handling dependencies")
emitter.assert_progress("Installing dependencies")
def test_build_dependencies_virtualenv_error_basicpip(tmp_path):
"""Process is properly interrupted if using pip fails."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=["something"],
)
with patch("charmcraft.charm_builder._process_run") as mock:
mock.return_value = -7
with pytest.raises(CommandError, match="problems using pip"):
builder.handle_dependencies()
def test_build_dependencies_virtualenv_none(tmp_path):
"""The virtualenv is NOT created if no needed."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=[],
)
with patch("charmcraft.charm_builder.subprocess.run") as mock_run:
builder.handle_dependencies()
mock_run.assert_not_called()
def test_build_dependencies_virtualenv_error_installing(tmp_path):
"""Process is properly interrupted if virtualenv creation fails."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=["something"],
)
with patch("charmcraft.charm_builder._process_run") as mock:
mock.side_effect = [0, -7]
with pytest.raises(CommandError,
match="problems installing dependencies"):
builder.handle_dependencies()
def test_build_dependencies_virtualenv_none(tmp_path, emitter):
"""The virtualenv is NOT created if no needed."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=[],
python_packages=[],
requirements=[],
)
with patch("charmcraft.charm_builder.subprocess.run") as mock_run:
builder.handle_dependencies()
mock_run.assert_not_called()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("No dependencies to handle")
def test_build_dependencies_virtualenv_simple(tmp_path):
"""A virtualenv is created with the specified requirements file."""
metadata = tmp_path / CHARM_METADATA
metadata.write_text("name: crazycharm")
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
requirements=["reqs.txt"],
)
with patch("charmcraft.charm_builder.subprocess.run") as mock_run:
mock_run.return_value.returncode = 1
with patch("charmcraft.charm_builder._process_run") as mock:
mock.return_value = 0
builder.handle_dependencies()
envpath = build_dir / VENV_DIRNAME
assert mock.mock_calls == [
call(["pip3", "--version"]),
call([
"pip3", "install", "--target={}".format(envpath),
"--requirement=reqs.txt"
]),
]
assert mock_run.mock_calls == [
call(
[
"python3",
"-c",
("from pip.commands.install import InstallCommand; "
'assert InstallCommand().cmd_opts.get_option("--system") is not None'
),
],
stdout=-3,
stderr=-3,
),
]
def test_build_dependencies_no_reused_problematic_hash_file(tmp_path, emitter):
"""Dependencies are built again because having problems to read the previous hash file."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=[],
python_packages=["ops"],
requirements=[],
)
staging_venv_dir = tmp_path / STAGING_VENV_DIRNAME
# patch the dependencies installation method so it skips all subprocessing but actually
# creates the directory, to simplify testing
builder._install_dependencies = lambda dirpath: dirpath.mkdir(exist_ok=True
)
# first run!
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("Dependencies directory not found")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed
assert staging_venv_dir.exists()
# installation directory copied to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
# avoid the file to be read succesfully
(tmp_path / DEPENDENCIES_HASH_FILENAME).write_bytes(
b"\xc3\x28") # invalid UTF8
# second run!
emitter.interactions.clear()
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace(
"Problems reading the dependencies hash file: "
"'utf-8' codec can't decode byte 0xc3 in position 0: invalid continuation byte"
)
emitter.assert_progress("Installing dependencies")
# directory created and packages installed *again*
assert staging_venv_dir.exists()
# installation directory copied *again* to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
def test_build_dependencies_no_reused_different_dependencies(
tmp_path, emitter, new_reqs_content, new_pypackages, new_pybinaries):
"""Dependencies are built again because changed from previous run."""
build_dir = tmp_path / BUILD_DIRNAME
build_dir.mkdir()
# prepare some dependencies for the first call (and some content for the second one)
reqs_file = tmp_path / "requirements.txt"
reqs_file.write_text("ops==1")
requirements = [reqs_file]
python_packages = ["foo", "bar"]
binary_python_packages = ["binthing"]
builder = CharmBuilder(
charmdir=tmp_path,
builddir=build_dir,
entrypoint=pathlib.Path("whatever"),
binary_python_packages=binary_python_packages,
python_packages=python_packages,
requirements=requirements,
)
staging_venv_dir = tmp_path / STAGING_VENV_DIRNAME
# patch the dependencies installation method so it skips all subprocessing but actually
# creates the directory, to simplify testing
builder._install_dependencies = lambda dirpath: dirpath.mkdir(exist_ok=True
)
# first run!
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_trace("Dependencies directory not found")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed
assert staging_venv_dir.exists()
# installation directory copied to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]
# for the second call, default new dependencies to first ones so only one is changed at a time
if new_reqs_content is not None:
reqs_file.write_text(new_reqs_content)
if new_pypackages is None:
new_pypackages = python_packages
if new_pybinaries is None:
new_pybinaries = binary_python_packages
# second run with other dependencies!
emitter.interactions.clear()
builder.binary_python_packages = new_pybinaries
builder.python_packages = new_pypackages
with patch("shutil.copytree") as mock_copytree:
builder.handle_dependencies()
emitter.assert_trace("Handling dependencies")
emitter.assert_progress("Installing dependencies")
# directory created and packages installed *again*
assert staging_venv_dir.exists()
# installation directory copied *again* to the build directory
site_packages_dir = charm_builder._find_venv_site_packages(
pathlib.Path(STAGING_VENV_DIRNAME))
assert mock_copytree.mock_calls == [
call(site_packages_dir, build_dir / VENV_DIRNAME)
]