def test_build_uses_custom_tag_when_given(machine: mock.Mock, architecture: str, foundation_file: str) -> None:
create_fake_repositories()
machine.return_value = architecture
docker_manager = mock.Mock()
docker_manager.build_image.side_effect = build_image
container.docker_manager.override(providers.Object(docker_manager))
result = CliRunner().invoke(lean, ["build", ".", "--tag", "my-tag"])
assert result.exit_code == 0
foundation_dockerfile = Path.cwd() / "Lean" / foundation_file
engine_dockerfile = Path.cwd() / "Lean" / "Dockerfile"
research_dockerfile = Path.cwd() / "Lean" / "DockerfileJupyter"
foundation_image = DockerImage(name="lean-cli/foundation", tag="my-tag")
engine_image = DockerImage(name="lean-cli/engine", tag="my-tag")
research_image = DockerImage(name="lean-cli/research", tag="my-tag")
docker_manager.build_image.assert_any_call(Path.cwd(), foundation_dockerfile, foundation_image)
docker_manager.build_image.assert_any_call(Path.cwd(), engine_dockerfile, engine_image)
docker_manager.build_image.assert_any_call(Path.cwd(), research_dockerfile, research_image)
assert f"""
FROM {foundation_image}
RUN true
""".strip() in dockerfiles_seen
assert f"""
FROM {engine_image}
RUN true
""".strip() in dockerfiles_seen
def build(root: Path, tag: str) -> None:
"""Build Docker images of your own version of LEAN and the Alpha Streams SDK.
\b
ROOT must point to a directory containing the LEAN repository and the Alpha Streams SDK repository:
https://github.com/QuantConnect/Lean & https://github.com/QuantConnect/AlphaStreams
When ROOT is not given, the current directory is used as root directory.
\b
This command performs the following actions:
1. The lean-cli/foundation:latest image is built from Lean/DockerfileLeanFoundation(ARM).
2. LEAN is compiled in a Docker container using the lean-cli/foundation:latest image.
3. The Alpha Streams SDK is compiled in a Docker container using the lean-cli/foundation:latest image.
4. The lean-cli/engine:latest image is built from Lean/Dockerfile using lean-cli/foundation:latest as base image.
5. The lean-cli/research:latest image is built from Lean/DockerfileJupyter using lean-cli/engine:latest as base image.
6. The default engine image is set to lean-cli/engine:latest.
7. The default research image is set to lean-cli/research:latest.
"""
lean_dir = root / "Lean"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/Lean to '{lean_dir}'"
)
alpha_streams_dir = root / "AlphaStreams"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/AlphaStreams to '{alpha_streams_dir}'"
)
(root / "DataLibraries").mkdir(exist_ok=True)
if platform.machine() in ["arm64", "aarch64"]:
foundation_dockerfile = lean_dir / "DockerfileLeanFoundationARM"
else:
foundation_dockerfile = lean_dir / "DockerfileLeanFoundation"
custom_foundation_image = DockerImage(name="lean-cli/foundation", tag=tag)
custom_engine_image = DockerImage(name="lean-cli/engine", tag=tag)
custom_research_image = DockerImage(name="lean-cli/research", tag=tag)
_build_image(root, foundation_dockerfile, None, custom_foundation_image)
_compile_csharp(root, lean_dir, custom_foundation_image)
_compile_csharp(root, alpha_streams_dir, custom_foundation_image)
_build_image(root, lean_dir / "Dockerfile", custom_foundation_image,
custom_engine_image)
_build_image(root, lean_dir / "DockerfileJupyter", custom_engine_image,
custom_research_image)
logger = container.logger()
cli_config_manager = container.cli_config_manager()
logger.info(f"Setting default engine image to '{custom_engine_image}'")
cli_config_manager.engine_image.set_value(str(custom_engine_image))
logger.info(f"Setting default research image to '{custom_research_image}'")
cli_config_manager.research_image.set_value(str(custom_research_image))
def test_live_passes_custom_image_to_lean_runner_when_given_as_option() -> None:
create_fake_lean_cli_directory()
create_fake_environment("live-paper", True)
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
lean_runner = mock.Mock()
container.lean_runner.override(providers.Object(lean_runner))
container.cli_config_manager().engine_image.set_value("custom/lean:123")
result = CliRunner().invoke(lean,
["live", "Python Project", "--environment", "live-paper", "--image", "custom/lean:456"])
assert result.exit_code == 0
lean_runner.run_lean.assert_called_once_with(mock.ANY,
"live-paper",
Path("Python Project/main.py").resolve(),
mock.ANY,
DockerImage(name="custom/lean", tag="456"),
None,
False,
False)
def test_warn_if_docker_image_outdated_does_nothing_when_latest_tag_not_installed(
requests_mock: RequestsMock) -> None:
logger, storage, docker_manager, update_manager = create_objects()
docker_manager.tag_installed.return_value = False
docker_manager.get_digest.return_value = "def"
update_manager.warn_if_docker_image_outdated(
DockerImage(name="quantconnect/lean", tag="latest"))
logger.warn.assert_not_called()
def _get_image_name(self, option: Option, default: str, override: Optional[str]) -> DockerImage:
"""Returns the image to use.
:param option: the CLI option that configures the image type
:param override: the image name to use, overriding any defaults or previously configured options
:param default: the default image to use when the option is not set and no override is given
:return: the image to use
"""
if override is not None:
image = override
else:
image = option.get_value(default)
return DockerImage.parse(image)
def test_research_runs_custom_image_when_given_as_option() -> None:
create_fake_lean_cli_directory()
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
container.cli_config_manager().research_image.set_value("custom/research:123")
result = CliRunner().invoke(lean, ["research", "Python Project", "--image", "custom/research:456"])
assert result.exit_code == 0
docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args
assert args[0] == DockerImage(name="custom/research", tag="456")
def test_warn_if_docker_image_outdated_does_nothing_when_not_outdated(
requests_mock: RequestsMock) -> None:
requests_mock.add(
requests_mock.GET,
"https://registry.hub.docker.com/v2/repositories/quantconnect/lean/tags/latest",
'{ "images": [ { "digest": "abc" } ] }')
logger, storage, docker_manager, update_manager = create_objects()
docker_manager.tag_installed.return_value = True
docker_manager.get_digest.return_value = "abc"
update_manager.warn_if_docker_image_outdated(
DockerImage(name="quantconnect/lean", tag="latest"))
logger.warn.assert_not_called()
def test_report_runs_custom_image_when_set_in_config() -> None:
docker_manager = mock.Mock()
docker_manager.run_image.side_effect = run_image
container.docker_manager.override(providers.Object(docker_manager))
container.cli_config_manager().engine_image.set_value("custom/lean:123")
result = CliRunner().invoke(lean, [
"report", "--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"
])
assert result.exit_code == 0
docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args
assert args[0] == DockerImage(name="custom/lean", tag="123")
def test_data_generate_runs_custom_image_when_set_in_config() -> None:
create_fake_lean_cli_directory()
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
container.cli_config_manager().engine_image.set_value("custom/lean:123")
result = CliRunner().invoke(
lean,
["data", "generate", "--start", "20200101", "--symbol-count", "1"])
assert result.exit_code == 0
docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args
assert args[0] == DockerImage(name="custom/lean", tag="123")
def test_optimize_runs_custom_image_when_set_in_config() -> None:
create_fake_lean_cli_directory()
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
container.cli_config_manager().engine_image.set_value("custom/lean:123")
Storage(str(Path.cwd() / "Python Project" / "config.json")).set(
"parameters", {"param1": "1"})
result = CliRunner().invoke(lean, ["optimize", "Python Project"])
assert result.exit_code == 0
docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args
assert args[0] == DockerImage(name="custom/lean", tag="123")
def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config(
) -> None:
create_fake_lean_cli_directory()
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
lean_runner = mock.Mock()
container.lean_runner.override(providers.Object(lean_runner))
container.cli_config_manager().engine_image.set_value("custom/lean:123")
result = CliRunner().invoke(lean, ["backtest", "Python Project"])
assert result.exit_code == 0
lean_runner.run_lean.assert_called_once_with(
"backtesting",
Path("Python Project/main.py").resolve(), mock.ANY,
DockerImage(name="custom/lean", tag="123"), None)
def test_warn_if_docker_image_outdated_only_checks_once_every_two_weeks(
requests_mock: RequestsMock, hours: int,
update_warning_expected: bool) -> None:
if update_warning_expected:
requests_mock.add(
requests_mock.GET,
"https://registry.hub.docker.com/v2/repositories/quantconnect/lean/tags/latest",
'{ "images": [ { "digest": "abc" } ] }')
logger, storage, docker_manager, update_manager = create_objects()
storage.set("last-update-check-my-image",
(datetime.now(tz=timezone.utc) -
timedelta(hours=hours)).timestamp())
docker_manager.tag_installed.return_value = True
docker_manager.get_digest.return_value = "def"
update_manager.warn_if_docker_image_outdated(
DockerImage(name="quantconnect/lean", tag="latest"))
if update_warning_expected:
logger.warn.assert_called()
else:
logger.warn.assert_not_called()
def test_docker_image_name_parse_parses_value(value: str, name: str,
tag: str) -> None:
result = DockerImage.parse(value)
assert result.name == name
assert result.tag == tag
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest import mock
import pytest
from click.testing import CliRunner
from dependency_injector import providers
from lean.commands import lean
from lean.container import container
from lean.models.docker import DockerImage
CUSTOM_FOUNDATION_IMAGE = DockerImage(name="lean-cli/foundation", tag="latest")
CUSTOM_ENGINE_IMAGE = DockerImage(name="lean-cli/engine", tag="latest")
CUSTOM_RESEARCH_IMAGE = DockerImage(name="lean-cli/research", tag="latest")
def create_fake_repositories() -> None:
lean_dir = Path.cwd() / "Lean"
alpha_streams_dir = Path.cwd() / "AlphaStreams"
for name in ["DockerfileLeanFoundation", "DockerfileLeanFoundationARM", "Dockerfile", "DockerfileJupyter"]:
path = lean_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w+", encoding="utf-8") as file:
file.write("""
FROM ubuntu
RUN true
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Tuple
from unittest import mock
import pytest
from docker.errors import APIError
from responses import RequestsMock
import lean
from lean.components.config.storage import Storage
from lean.components.util.http_client import HTTPClient
from lean.components.util.update_manager import UpdateManager
from lean.models.docker import DockerImage
DOCKER_IMAGE = DockerImage(name="quantconnect/lean", tag="latest")
def create_objects() -> Tuple[mock.Mock, Storage, mock.Mock, UpdateManager]:
logger = mock.Mock()
storage = Storage(str(Path("~/.lean/cache").expanduser()))
docker_manager = mock.Mock()
update_manager = UpdateManager(logger, HTTPClient(logger), storage,
docker_manager)
return logger, storage, docker_manager, update_manager
@mock.patch.object(lean, "__version__", "0.0.1")
def test_warn_if_cli_outdated_warns_when_pypi_version_newer_than_current_version(
import pytest
from click.testing import CliRunner
from dependency_injector import providers
from lean.commands import lean
from lean.components.config.storage import Storage
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container
from lean.models.docker import DockerImage
from lean.models.optimizer import (OptimizationConstraint,
OptimizationExtremum, OptimizationParameter,
OptimizationTarget)
from tests.test_helpers import create_fake_lean_cli_directory
ENGINE_IMAGE = DockerImage.parse(DEFAULT_ENGINE_IMAGE)
@pytest.fixture(autouse=True)
def update_manager_mock() -> mock.Mock:
"""A pytest fixture which mocks the update manager before every test."""
update_manager = mock.Mock()
container.update_manager.override(providers.Object(update_manager))
return update_manager
@pytest.fixture(autouse=True)
def optimizer_config_manager_mock() -> mock.Mock:
"""A pytest fixture which mocks the optimizer config manager before every test."""
optimizer_config_manager = mock.Mock()
optimizer_config_manager.configure_strategy.return_value = "QuantConnect.Optimizer.Strategies.GridSearchOptimizationStrategy"
import json
from pathlib import Path
from typing import Optional
from unittest import mock
import pytest
from click.testing import CliRunner
from dependency_injector import providers
from lean.commands import lean
from lean.constants import DEFAULT_RESEARCH_IMAGE
from lean.container import container
from lean.models.docker import DockerImage
from tests.test_helpers import create_fake_lean_cli_directory
RESEARCH_IMAGE = DockerImage.parse(DEFAULT_RESEARCH_IMAGE)
@pytest.fixture(autouse=True)
def update_manager_mock() -> mock.Mock:
"""A pytest fixture which mocks the update manager before every test."""
update_manager = mock.Mock()
container.update_manager.override(providers.Object(update_manager))
return update_manager
def test_research_runs_research_container() -> None:
create_fake_lean_cli_directory()
docker_manager = mock.Mock()
container.docker_manager.override(providers.Object(docker_manager))
def test_get_research_image_returns_override_when_given() -> None:
cli_config_manager = CLIConfigManager(create_storage(), create_storage())
cli_config_manager.research_image.set_value("custom/research:3")
assert cli_config_manager.get_research_image(
"custom/research:5") == DockerImage(name="custom/research", tag="5")
def test_get_research_image_returns_image_configured_via_option() -> None:
cli_config_manager = CLIConfigManager(create_storage(), create_storage())
cli_config_manager.research_image.set_value("custom/research:3")
assert cli_config_manager.get_research_image() == DockerImage(
name="custom/research", tag="3")
def test_get_research_image_returns_default_image_when_nothing_configured(
) -> None:
cli_config_manager = CLIConfigManager(create_storage(), create_storage())
assert cli_config_manager.get_research_image() == DockerImage.parse(
DEFAULT_RESEARCH_IMAGE)
def test_docker_image_str_returns_full_name() -> None:
assert str(DockerImage(name="quantconnect/lean",
tag="latest")) == "quantconnect/lean:latest"
def start(organization: Optional[str], port: int, no_open: bool,
shortcut: bool, gui: Optional[Path], shortcut_launch: bool) -> None:
"""Start the local GUI."""
logger = container.logger()
docker_manager = container.docker_manager()
temp_manager = container.temp_manager()
module_manager = container.module_manager()
api_client = container.api_client()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is not None:
if gui_container.status == "running":
if shortcut_launch:
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
webbrowser.open(url)
return
else:
_error(
"The local GUI is already running, run `lean gui restart` to restart it or `lean gui stop` to stop it",
shortcut_launch)
gui_container.remove()
if organization is not None:
organization_id = _get_organization_id(organization, shortcut_launch)
else:
organizations = api_client.organizations.get_all()
options = [
Option(id=organization.id, label=organization.name)
for organization in organizations
]
organization_id = logger.prompt_list(
"Select the organization with the local GUI module subscription",
options)
module_manager.install_module(GUI_PRODUCT_INSTALL_ID, organization_id)
shortcut_manager = container.shortcut_manager()
if shortcut:
shortcut_manager.create_shortcut(organization_id)
else:
shortcut_manager.prompt_if_necessary(organization_id)
# The dict containing all options passed to `docker run`
# See all available options at https://docker-py.readthedocs.io/en/stable/containers.html
run_options: Dict[str, Any] = {
"name": LOCAL_GUI_CONTAINER_NAME,
"detach": True,
"remove": False,
"commands": [],
"environment": {
"PYTHONUNBUFFERED": "1",
"QC_LOCAL_GUI": "true",
"QC_DOCKER_HOST_SYSTEM": platform.system(),
"QC_DOCKER_HOST_MACHINE": platform.machine(),
"QC_ORGANIZATION_ID": organization_id,
"QC_API": os.environ.get("QC_API", "")
},
"mounts": [],
"volumes": {},
"ports": {
"5612": str(port)
}
}
# Cache the site-packages so we don't re-install everything when the container is restarted
docker_manager.create_volume("lean_cli_gui_python")
run_options["volumes"]["lean_cli_gui_python"] = {
"bind": "/root/.local/lib/python3.9/site-packages",
"mode": "rw"
}
# Update PATH in the GUI container to add executables installed with pip
run_options["commands"].append('export PATH="$PATH:/root/.local/bin"')
package_file_name = module_manager.get_installed_packages_by_module(
GUI_PRODUCT_INSTALL_ID)[0].get_file_name()
with zipfile.ZipFile(Path.home() / ".lean" / "modules" /
package_file_name) as package_file:
content_file_names = [
f.replace("content/", "") for f in package_file.namelist()
if f.startswith("content/")
]
wheel_file_name = next(f for f in content_file_names
if f.endswith(".whl"))
terminal_file_name = next(f for f in content_file_names
if f.endswith(".zip"))
# Install the CLI in the GUI container
run_options["commands"].append("pip uninstall -y lean")
if lean.__version__ == "dev":
lean_cli_dir = str(
Path(__file__).absolute().parent.parent.parent.parent)
logger.info(
f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli"
)
run_options["volumes"][str(lean_cli_dir)] = {
"bind": "/lean-cli",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
else:
run_options["commands"].append(
"pip install --user --progress-bar off --upgrade lean")
# Install the GUI in the GUI container
run_options["commands"].append("pip uninstall -y leangui")
if gui is None:
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{wheel_file_name} > /{wheel_file_name}"
)
run_options["commands"].append(
f"pip install --user --progress-bar off /{wheel_file_name}")
elif gui.is_file():
run_options["mounts"].append(
Mount(target=f"/{gui.name}",
source=str(gui),
type="bind",
read_only=True))
run_options["commands"].append(
f"pip install --user --progress-bar off /{gui.name}")
else:
run_options["volumes"][str(gui)] = {
"bind": "/lean-cli-gui",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli-gui")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
# Extract the terminal in the GUI container
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{terminal_file_name} > /{terminal_file_name}"
)
run_options["commands"].append(
f"unzip -o /{terminal_file_name} -d /terminal")
# Write correct streaming url to /terminal/local.socket.host.conf
run_options["commands"].append(
f'echo "ws://localhost:{port}/streaming" > /terminal/local.socket.host.conf'
)
# Mount the `lean init` directory in the GUI container
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
run_options["volumes"][str(cli_root_dir)] = {
"bind": "/LeanCLI",
"mode": "rw"
}
# Mount the global config directory in the GUI container
run_options["volumes"][str(Path("~/.lean").expanduser())] = {
"bind": "/root/.lean",
"mode": "rw"
}
# Mount a directory to the tmp directory in the GUI container
gui_tmp_directory = temp_manager.create_temporary_directory()
run_options["volumes"][str(gui_tmp_directory)] = {
"bind": "/tmp",
"mode": "rw"
}
# Set up the path mappings between paths in the host system and paths in the GUI container
run_options["environment"]["DOCKER_PATH_MAPPINGS"] = json.dumps({
"/LeanCLI":
cli_root_dir.as_posix(),
"/root/.lean":
Path("~/.lean").expanduser().as_posix(),
"/tmp":
gui_tmp_directory.as_posix()
})
# Mount the Docker socket in the GUI container
run_options["mounts"].append(
Mount(target="/var/run/docker.sock",
source="/var/run/docker.sock",
type="bind",
read_only=False))
# Run the GUI in the GUI container
run_options["commands"].append("cd /LeanCLI")
run_options["commands"].append(f"leangui")
# Don't delete temporary directories when the command exits, the container will still need them
temp_manager.delete_temporary_directories_when_done = False
logger.info("Starting the local GUI, this may take some time...")
# Pull the Docker images used by the local GUI
# If this is done while the local GUI is running there is a big delay between pressing Backtest and seeing it run
update_manager = container.update_manager()
cli_config_manager = container.cli_config_manager()
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_engine_image(), False)
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_research_image(), False)
try:
docker_manager.run_image(
DockerImage(name="python", tag="3.9.6-buster"), **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
_error(
f"Port {port} is already in use, please specify a different port using --port <number>",
shortcut_launch)
raise error
url = f"http://localhost:{port}/"
# Wait until the GUI is running
while True:
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
if shortcut_launch:
_error(
"Something went wrong while starting the local GUI, run `lean gui logs` for more information",
shortcut_launch)
else:
_error(
"Something went wrong while starting the local GUI, see the logs above for more information",
shortcut_launch)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
logger.info(f"The local GUI has started and is running on {url}")
if not no_open:
webbrowser.open(url)
