def _build_image(dockerfile: Path, base_image: Optional[DockerImage],
target_image: DockerImage) -> None:
"""Builds a Docker image.
:param dockerfile: the path to the Dockerfile to build
:param base_image: the base image to use, or None if the default should be used
:param target_image: the name of the new image
"""
logger = container.logger()
if base_image is not None:
logger.info(
f"Building '{target_image}' from '{dockerfile}' using '{base_image}' as base image"
)
else:
logger.info(f"Building '{target_image}' from '{dockerfile}'")
if not dockerfile.is_file():
raise RuntimeError(f"'{dockerfile}' does not exist")
current_content = dockerfile.read_text(encoding="utf-8")
if base_image is not None:
new_content = re.sub(r"^FROM.*$",
f"FROM {base_image}",
current_content,
flags=re.MULTILINE)
dockerfile.write_text(new_content, encoding="utf-8")
try:
docker_manager = container.docker_manager()
docker_manager.build_image(dockerfile, target_image)
finally:
if base_image is not None:
dockerfile.write_text(current_content, encoding="utf-8")
def _compile_lean(lean_dir: Path) -> None:
"""Compiles LEAN's C# code.
:param lean_dir: the directory containing the LEAN repository
"""
logger = container.logger()
logger.info(f"Compiling the C# code in '{lean_dir}'")
docker_manager = container.docker_manager()
docker_manager.create_volume("lean_cli_nuget")
success = docker_manager.run_image(
CUSTOM_FOUNDATION_IMAGE,
entrypoint=["dotnet", "build", f"/LeanCLI"],
environment={
"DOTNET_CLI_TELEMETRY_OPTOUT": "true",
"DOTNET_NOLOGO": "true"
},
volumes={
str(lean_dir): {
"bind": "/LeanCLI",
"mode": "rw"
},
"lean_cli_nuget": {
"bind": "/root/.nuget/packages",
"mode": "rw"
}
})
if not success:
raise RuntimeError(
"Something went wrong while running dotnet build, see the logs above for more information"
)
def live(project: Path, environment: str, output: Optional[Path],
image: Optional[str], update: bool) -> None:
"""Start live trading a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
ENVIRONMENT must be the name of an environment in the Lean configuration file with live-mode set to true.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
if output is None:
output = algorithm_file.parent / "live" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
environment, algorithm_file, None)
if "environments" not in lean_config or environment not in lean_config[
"environments"]:
lean_config_path = lean_config_manager.get_lean_config_path()
raise MoreInfoError(
f"{lean_config_path} does not contain an environment named '{environment}'",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
if not lean_config["environments"][environment]["live-mode"]:
raise MoreInfoError(
f"The '{environment}' is not a live trading environment (live-mode is set to false)",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
_raise_for_missing_properties(lean_config, environment,
lean_config_manager.get_lean_config_path())
_start_iqconnect_if_necessary(lean_config, environment)
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_runner = container.lean_runner()
lean_runner.run_lean(environment, algorithm_file, output, engine_image,
None)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def logs(follow: bool) -> None:
"""See the logs of the local GUI."""
docker_manager = container.docker_manager()
if docker_manager.get_container_by_name(LOCAL_GUI_CONTAINER_NAME) is None:
raise RuntimeError("The local GUI container does not exist, you can start it using `lean gui start`")
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME, follow)
def backtest(project: Path, output: Optional[Path], debug: Optional[str],
image: Optional[str], update: bool) -> None:
"""Backtest a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
Go to the following url to learn how to debug backtests locally using the Lean CLI:
https://www.quantconnect.com/docs/v2/lean-cli/tutorials/backtesting/debugging-local-backtests
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
if output is None:
output = algorithm_file.parent / "backtests" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
debugging_method = None
if debug == "pycharm":
debugging_method = DebuggingMethod.PyCharm
_migrate_dotnet_5_python_pycharm(algorithm_file.parent)
elif debug == "ptvsd":
debugging_method = DebuggingMethod.PTVSD
_migrate_dotnet_5_python_vscode(algorithm_file.parent)
elif debug == "vsdbg":
debugging_method = DebuggingMethod.VSDBG
_migrate_dotnet_5_csharp_vscode(algorithm_file.parent)
elif debug == "rider":
debugging_method = DebuggingMethod.Rider
_migrate_dotnet_5_csharp_rider(algorithm_file.parent)
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_runner = container.lean_runner()
lean_runner.run_lean("backtesting", algorithm_file, output, engine_image,
debugging_method)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def stop() -> None:
"""Stop the local GUI."""
docker_manager = container.docker_manager()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
raise RuntimeError(
"The local GUI is not running, you can start it using `lean gui start`"
)
gui_container.stop()
for container_name in docker_manager.get_running_containers():
if container_name.startswith("lean_cli_gui_research_"):
docker_manager.get_container_by_name(container_name).stop()
container.logger().info(f"Successfully stopped the local GUI")
def invoke(self, ctx):
if self._requires_lean_config:
try:
# This method will throw if the directory cannot be found
container.lean_config_manager().get_cli_root_directory()
except Exception:
# Abort with a display-friendly error message if the command requires a Lean config
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/user-guides/troubleshooting#02-Common-errors")
if self._requires_docker and "pytest" not in sys.modules:
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if platform.system() == "Linux":
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if platform.system() == "Linux" and os.getuid() != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password")
args = ["sudo", "--preserve-env=HOME", sys.executable, *sys.argv]
os.execlp(args[0], *args)
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def restart(no_open: bool) -> None:
"""Restart the local GUI and open it in the browser."""
logger = container.logger()
docker_manager = container.docker_manager()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
raise RuntimeError(
"The local GUI is not running, you can start it using `lean gui start`"
)
logger.info("Restarting the local GUI's Docker container")
gui_container.restart()
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
# Wait until the GUI is running again
while True:
if LOCAL_GUI_CONTAINER_NAME not in docker_manager.get_running_containers(
):
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
raise RuntimeError(
"Something went wrong while restarting the local GUI, see the logs above for more information"
)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
container.logger().info(
f"The local GUI has restarted and is running on {url}")
if not no_open:
webbrowser.open(url)
def _compile_csharp(root: Path, csharp_dir: Path,
docker_image: DockerImage) -> None:
"""Compiles C# code.
:param root: the root directory in which the command is ran
:param csharp_dir: the directory containing the C# code
:param docker_image: the Docker image to compile in
"""
logger = container.logger()
logger.info(f"Compiling the C# code in '{csharp_dir}'")
build_path = Path("/LeanCLI") / csharp_dir.relative_to(root)
docker_manager = container.docker_manager()
docker_manager.create_volume("lean_cli_nuget")
success = docker_manager.run_image(
docker_image,
entrypoint=["dotnet", "build", str(build_path)],
environment={
"DOTNET_CLI_TELEMETRY_OPTOUT": "true",
"DOTNET_NOLOGO": "true"
},
volumes={
str(root): {
"bind": "/LeanCLI",
"mode": "rw"
},
"lean_cli_nuget": {
"bind": "/root/.nuget/packages",
"mode": "rw"
}
})
if not success:
raise RuntimeError(
"Something went wrong while running dotnet build, see the logs above for more information"
)
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options = {
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
},
str(project): {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_run": lambda: webbrowser.open(f"http://localhost:{port}/")
}
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
if "port is already allocated" in error.explanation:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
def start(organization: Optional[str], port: int, no_open: bool,
shortcut: bool, gui: Optional[Path], shortcut_launch: bool) -> None:
"""Start the local GUI."""
logger = container.logger()
docker_manager = container.docker_manager()
temp_manager = container.temp_manager()
module_manager = container.module_manager()
api_client = container.api_client()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is not None:
if gui_container.status == "running":
if shortcut_launch:
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
webbrowser.open(url)
return
else:
_error(
"The local GUI is already running, run `lean gui restart` to restart it or `lean gui stop` to stop it",
shortcut_launch)
gui_container.remove()
if organization is not None:
organization_id = _get_organization_id(organization, shortcut_launch)
else:
organizations = api_client.organizations.get_all()
options = [
Option(id=organization.id, label=organization.name)
for organization in organizations
]
organization_id = logger.prompt_list(
"Select the organization with the local GUI module subscription",
options)
module_manager.install_module(GUI_PRODUCT_INSTALL_ID, organization_id)
shortcut_manager = container.shortcut_manager()
if shortcut:
shortcut_manager.create_shortcut(organization_id)
else:
shortcut_manager.prompt_if_necessary(organization_id)
# The dict containing all options passed to `docker run`
# See all available options at https://docker-py.readthedocs.io/en/stable/containers.html
run_options: Dict[str, Any] = {
"name": LOCAL_GUI_CONTAINER_NAME,
"detach": True,
"remove": False,
"commands": [],
"environment": {
"PYTHONUNBUFFERED": "1",
"QC_LOCAL_GUI": "true",
"QC_DOCKER_HOST_SYSTEM": platform.system(),
"QC_DOCKER_HOST_MACHINE": platform.machine(),
"QC_ORGANIZATION_ID": organization_id,
"QC_API": os.environ.get("QC_API", "")
},
"mounts": [],
"volumes": {},
"ports": {
"5612": str(port)
}
}
# Cache the site-packages so we don't re-install everything when the container is restarted
docker_manager.create_volume("lean_cli_gui_python")
run_options["volumes"]["lean_cli_gui_python"] = {
"bind": "/root/.local/lib/python3.9/site-packages",
"mode": "rw"
}
# Update PATH in the GUI container to add executables installed with pip
run_options["commands"].append('export PATH="$PATH:/root/.local/bin"')
package_file_name = module_manager.get_installed_packages_by_module(
GUI_PRODUCT_INSTALL_ID)[0].get_file_name()
with zipfile.ZipFile(Path.home() / ".lean" / "modules" /
package_file_name) as package_file:
content_file_names = [
f.replace("content/", "") for f in package_file.namelist()
if f.startswith("content/")
]
wheel_file_name = next(f for f in content_file_names
if f.endswith(".whl"))
terminal_file_name = next(f for f in content_file_names
if f.endswith(".zip"))
# Install the CLI in the GUI container
run_options["commands"].append("pip uninstall -y lean")
if lean.__version__ == "dev":
lean_cli_dir = str(
Path(__file__).absolute().parent.parent.parent.parent)
logger.info(
f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli"
)
run_options["volumes"][str(lean_cli_dir)] = {
"bind": "/lean-cli",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
else:
run_options["commands"].append(
"pip install --user --progress-bar off --upgrade lean")
# Install the GUI in the GUI container
run_options["commands"].append("pip uninstall -y leangui")
if gui is None:
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{wheel_file_name} > /{wheel_file_name}"
)
run_options["commands"].append(
f"pip install --user --progress-bar off /{wheel_file_name}")
elif gui.is_file():
run_options["mounts"].append(
Mount(target=f"/{gui.name}",
source=str(gui),
type="bind",
read_only=True))
run_options["commands"].append(
f"pip install --user --progress-bar off /{gui.name}")
else:
run_options["volumes"][str(gui)] = {
"bind": "/lean-cli-gui",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli-gui")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
# Extract the terminal in the GUI container
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{terminal_file_name} > /{terminal_file_name}"
)
run_options["commands"].append(
f"unzip -o /{terminal_file_name} -d /terminal")
# Write correct streaming url to /terminal/local.socket.host.conf
run_options["commands"].append(
f'echo "ws://localhost:{port}/streaming" > /terminal/local.socket.host.conf'
)
# Mount the `lean init` directory in the GUI container
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
run_options["volumes"][str(cli_root_dir)] = {
"bind": "/LeanCLI",
"mode": "rw"
}
# Mount the global config directory in the GUI container
run_options["volumes"][str(Path("~/.lean").expanduser())] = {
"bind": "/root/.lean",
"mode": "rw"
}
# Mount a directory to the tmp directory in the GUI container
gui_tmp_directory = temp_manager.create_temporary_directory()
run_options["volumes"][str(gui_tmp_directory)] = {
"bind": "/tmp",
"mode": "rw"
}
# Set up the path mappings between paths in the host system and paths in the GUI container
run_options["environment"]["DOCKER_PATH_MAPPINGS"] = json.dumps({
"/LeanCLI":
cli_root_dir.as_posix(),
"/root/.lean":
Path("~/.lean").expanduser().as_posix(),
"/tmp":
gui_tmp_directory.as_posix()
})
# Mount the Docker socket in the GUI container
run_options["mounts"].append(
Mount(target="/var/run/docker.sock",
source="/var/run/docker.sock",
type="bind",
read_only=False))
# Run the GUI in the GUI container
run_options["commands"].append("cd /LeanCLI")
run_options["commands"].append(f"leangui")
# Don't delete temporary directories when the command exits, the container will still need them
temp_manager.delete_temporary_directories_when_done = False
logger.info("Starting the local GUI, this may take some time...")
# Pull the Docker images used by the local GUI
# If this is done while the local GUI is running there is a big delay between pressing Backtest and seeing it run
update_manager = container.update_manager()
cli_config_manager = container.cli_config_manager()
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_engine_image(), False)
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_research_image(), False)
try:
docker_manager.run_image(
DockerImage(name="python", tag="3.9.6-buster"), **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
_error(
f"Port {port} is already in use, please specify a different port using --port <number>",
shortcut_launch)
raise error
url = f"http://localhost:{port}/"
# Wait until the GUI is running
while True:
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
if shortcut_launch:
_error(
"Something went wrong while starting the local GUI, run `lean gui logs` for more information",
shortcut_launch)
else:
_error(
"Something went wrong while starting the local GUI, see the logs above for more information",
shortcut_launch)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
logger.info(f"The local GUI has started and is running on {url}")
if not no_open:
webbrowser.open(url)
def report(backtest_data_source_file: Path,
live_data_source_file: Optional[Path],
report_destination: Path,
strategy_name: Optional[str],
strategy_version: Optional[str],
strategy_description: Optional[str],
overwrite: bool,
image: Optional[str],
update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it")
project_directory = _find_project_directory(backtest_data_source_file)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder": "/Lean/Data",
"strategy-name": strategy_name or "",
"strategy-version": strategy_version or "",
"strategy-description": strategy_description or "",
"live-data-source-file": "live-data-source-file.json" if live_data_source_file is not None else "",
"backtest-data-source-file": "backtest-data-source-file.json",
"report-destination": "/Results/report.html",
"environment": "report",
"log-handler": "QuantConnect.Logging.CompositeLogHandler",
"messaging-handler": "QuantConnect.Messaging.Messaging",
"job-queue-handler": "QuantConnect.Queues.JobQueue",
"api-handler": "QuantConnect.Api.Api",
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler": "QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider": "DataChannelProvider",
"environments": {
"report": {
"live-mode": False,
"setup-handler": "QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler": "QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler": "QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider": "QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
output_dir = container.temp_manager().create_temporary_directory()
config_path = output_dir / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"working_dir": "/Lean/Report/bin/Debug",
"entrypoint": ["dotnet", "QuantConnect.Report.dll"],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_data_source_file),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "ro"
},
str(output_dir): {
"bind": "/Results",
"mode": "rw"
}
}
}
if live_data_source_file is not None:
run_options["mounts"].append(Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_data_source_file),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information")
report_destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output_dir / "report.html", report_destination)
logger = container.logger()
logger.info(f"Successfully generated report to '{report_destination}'")
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def report(backtest_results: Optional[Path], live_results: Optional[Path],
report_destination: Path, detach: bool,
strategy_name: Optional[str], strategy_version: Optional[str],
strategy_description: Optional[str], overwrite: bool,
image: Optional[str], update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
If --backtest-results is not given, a report is generated for the most recent local backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(
f"{report_destination} already exists, use --overwrite to overwrite it"
)
if backtest_results is None:
backtest_json_files = list(Path.cwd().rglob("backtests/*/*.json"))
result_json_files = [
f for f in backtest_json_files
if not f.name.endswith("-order-events.json")
and not f.name.endswith("alpha-results.json")
]
if len(result_json_files) == 0:
raise MoreInfoError(
"Could not find a recent backtest result file, please use the --backtest-results option",
"https://www.lean.io/docs/lean-cli/backtesting/report#02-Generate-a-Report"
)
backtest_results = sorted(result_json_files,
key=lambda f: f.stat().st_mtime,
reverse=True)[0]
logger = container.logger()
if live_results is None:
logger.info(f"Generating a report from '{backtest_results}'")
else:
logger.info(
f"Generating a report from '{backtest_results}' and '{live_results}'"
)
project_directory = _find_project_directory(backtest_results)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder":
"/Lean/Data",
"strategy-name":
strategy_name or "",
"strategy-version":
strategy_version or "",
"strategy-description":
strategy_description or "",
"live-data-source-file":
"live-data-source-file.json" if live_results is not None else "",
"backtest-data-source-file":
"backtest-data-source-file.json",
"report-destination":
"/tmp/report.html",
"environment":
"report",
"log-handler":
"QuantConnect.Logging.CompositeLogHandler",
"messaging-handler":
"QuantConnect.Messaging.Messaging",
"job-queue-handler":
"QuantConnect.Queues.JobQueue",
"api-handler":
"QuantConnect.Api.Api",
"map-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider":
"QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler":
"QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider":
"DataChannelProvider",
"environments": {
"report": {
"live-mode":
False,
"setup-handler":
"QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler":
"QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler":
"QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler":
"QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider":
"QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler":
"QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
config_path = container.temp_manager().create_temporary_directory(
) / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
backtest_id = container.output_config_manager().get_backtest_id(
backtest_results.parent)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
report_destination.parent.mkdir(parents=True, exist_ok=True)
run_options: Dict[str, Any] = {
"detach":
detach,
"name":
f"lean_cli_report_{backtest_id}",
"working_dir":
"/Lean/Report/bin/Debug",
"commands": [
"dotnet QuantConnect.Report.dll",
f'cp /tmp/report.html "/Output/{report_destination.name}"'
],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(
target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_results),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
},
str(report_destination.parent): {
"bind": "/Output",
"mode": "rw"
}
}
}
if live_results is not None:
run_options["mounts"].append(
Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_results),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image_override = image
if engine_image_override is None and project_directory is not None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
engine_image_override = project_config.get("engine-image", None)
engine_image = cli_config_manager.get_engine_image(engine_image_override)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information"
)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started the report creator in the '{run_options['name']}' container"
)
logger.info(f"The report will be generated to '{report_destination}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
return
logger.info(f"Successfully generated report to '{report_destination}'")
def generate(start: datetime, end: datetime, symbol_count: int,
security_type: str, resolution: str, data_density: str,
include_coarse: bool, market: str, image: Optional[str],
update: bool) -> None:
"""Generate random market data.
\b
This uses the random data generator in LEAN to generate realistic market data using a Brownian motion model.
This generator supports the following security types, tick types and resolutions:
| Security type | Generated tick types | Supported resolutions |
| ------------- | -------------------- | ------------------------------------ |
| Equity | Trade | Tick, Second, Minute, Hour and Daily |
| Forex | Quote | Tick, Second, Minute, Hour and Daily |
| CFD | Quote | Tick, Second, Minute, Hour and Daily |
| Future | Trade and Quote | Tick, Second, Minute, Hour and Daily |
| Crypto | Trade and Quote | Tick, Second, Minute, Hour and Daily |
| Option | Trade and Quote | Minute |
\b
The following data densities are available:
- Dense: at least one data point per resolution step.
- Sparse: at least one data point per 5 resolution steps.
- VerySparse: at least one data point per 50 resolution steps.
\b
Example which generates minute data for 100 equity symbols since 2015-01-01:
$ lean data generate --start=20150101 --symbol-count=100
\b
Example which generates daily data for 100 crypto symbols since 2015-01-01:
$ lean data generate --start=20150101 --symbol-count=100 --security-type=Crypto --resolution=Daily
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options = {
"entrypoint": [
"dotnet", "QuantConnect.ToolBox.dll", "--destination-dir",
"/Lean/Data", "--app", "randomdatagenerator", "--start",
start.strftime("%Y%m%d"), "--end",
end.strftime("%Y%m%d"), "--symbol-count",
str(symbol_count), "--security-type", security_type,
"--resolution", resolution, "--data-density", data_density,
"--include-coarse",
str(include_coarse).lower(), "--market",
market.lower()
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
}
}
}
engine_image = container.cli_config_manager().get_engine_image(image)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the random data generator, see the logs above for more information"
)
def invoke(self, ctx: click.Context):
if self._requires_lean_config:
lean_config_manager = container.lean_config_manager()
try:
# This method will raise an error if the directory cannot be found
lean_config_manager.get_cli_root_directory()
except Exception:
# Use one of the cached Lean config locations to avoid having to abort the command
lean_config_paths = lean_config_manager.get_known_lean_config_paths(
)
if len(lean_config_paths) > 0:
lean_config_path = container.logger().prompt_list(
"Select the Lean configuration file to use", [
Option(id=p, label=str(p))
for p in lean_config_paths
])
lean_config_manager.set_default_lean_config_path(
lean_config_path)
else:
# Abort with a display-friendly error message if the command requires a Lean config and none found
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors"
)
if self._requires_docker and "pytest" not in sys.modules:
is_system_linux = container.platform_manager().is_system_linux()
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if is_system_linux:
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if is_system_linux and os.getuid(
) != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password"
)
args = [
"sudo", "--preserve-env=HOME", sys.executable, *sys.argv
]
os.execlp(args[0], *args)
if self._allow_unknown_options:
# Unknown options are passed to ctx.args and need to be parsed manually
# We parse them to ctx.params so they're available like normal options
# Because of this all commands with allow_unknown_options=True must have a **kwargs argument
arguments = list(
itertools.chain(*[arg.split("=") for arg in ctx.args]))
skip_next = False
for index in range(len(arguments) - 1):
if skip_next:
skip_next = False
continue
if arguments[index].startswith("--"):
option = arguments[index].replace("--", "")
value = arguments[index + 1]
ctx.params[option] = value
skip_next = True
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def research(project: Path, port: int, data_provider: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
detach: bool, no_open: bool, image: Optional[str],
update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
lean_config["composer-dll-directory"] = "/Lean/Launcher/bin/Debug"
if download_data:
data_provider = QuantConnectDataProvider.get_name()
if data_provider is not None:
data_provider = next(dp for dp in all_data_providers
if dp.get_name() == data_provider)
data_provider.build(lean_config, container.logger()).configure(
lean_config, "backtesting")
lean_config_manager.configure_data_purchase_limit(lean_config,
data_purchase_limit)
lean_runner = container.lean_runner()
temp_manager = container.temp_manager()
run_options = lean_runner.get_basic_docker_config(
lean_config, algorithm_file, temp_manager.create_temporary_directory(),
None, False, detach)
# Mount the config in the notebooks directory as well
local_config_path = next(m["Source"] for m in run_options["mounts"]
if m["Target"].endswith("config.json"))
run_options["mounts"].append(
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(local_config_path),
type="bind",
read_only=True))
# Jupyter Lab runs on port 8888, we expose it to the local port specified by the user
run_options["ports"]["8888"] = str(port)
# Open the browser as soon as Jupyter Lab has started
if detach or not no_open:
run_options["on_output"] = lambda chunk: _check_docker_output(
chunk, port)
# Give container an identifiable name when running it from the GUI
if container.module_manager().is_module_installed(GUI_PRODUCT_INSTALL_ID):
project_id = container.project_config_manager().get_local_id(
algorithm_file.parent)
run_options["name"] = f"lean_cli_gui_research_{project_id}"
# Make Ctrl+C stop Jupyter Lab immediately
run_options["stop_signal"] = "SIGKILL"
# Mount the project to the notebooks directory
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Allow notebooks to be embedded in iframes
run_options["commands"].append("mkdir -p ~/.jupyter")
run_options["commands"].append(
'echo "c.NotebookApp.disable_check_xsrf = True\nc.NotebookApp.tornado_settings = {\'headers\': {\'Content-Security-Policy\': \'frame-ancestors self *\'}}" > ~/.jupyter/jupyter_notebook_config.py'
)
# Hide headers in notebooks
run_options["commands"].append(
"mkdir -p ~/.ipython/profile_default/static/custom")
run_options["commands"].append(
'echo "#header-container { display: none !important; }" > ~/.ipython/profile_default/static/custom/custom.css'
)
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
research_image = cli_config_manager.get_research_image(
image or project_config.get("research-image", None))
container.update_manager().pull_docker_image_if_necessary(
research_image, update)
try:
container.docker_manager().run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
raise RuntimeError(
f"Port {port} is already in use, please specify a different port using --port <number>"
)
raise error
if detach:
temp_manager.delete_temporary_directories_when_done = False
logger = container.logger()
relative_project_dir = algorithm_file.parent.relative_to(
lean_config_manager.get_cli_root_directory())
logger.info(
f"Successfully started Jupyter Lab environment for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(
"You can use Docker's own commands to manage the detached container"
)
def backtest(project: Path, output: Optional[Path], debug: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
image: Optional[str], update: bool) -> None:
"""Backtest a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
Go to the following url to learn how to debug backtests locally using the Lean CLI:
https://www.lean.io/docs/lean-cli/tutorials/backtesting/debugging-local-backtests
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
lean_config_manager = container.lean_config_manager()
if output is None:
output = algorithm_file.parent / "backtests" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
debugging_method = None
if debug == "pycharm":
debugging_method = DebuggingMethod.PyCharm
_migrate_dotnet_5_python_pycharm(algorithm_file.parent)
elif debug == "ptvsd":
debugging_method = DebuggingMethod.PTVSD
_migrate_dotnet_5_python_vscode(algorithm_file.parent)
elif debug == "vsdbg":
debugging_method = DebuggingMethod.VSDBG
_migrate_dotnet_5_csharp_vscode(algorithm_file.parent)
elif debug == "rider":
debugging_method = DebuggingMethod.Rider
_migrate_dotnet_5_csharp_rider(algorithm_file.parent)
if download_data:
organization = _select_organization()
lean_config_manager.set_property("job-organization-id",
organization.id)
lean_config_manager.set_property(
"data-provider",
"QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider")
lean_config_manager.set_property(
"map-file-provider",
"QuantConnect.Data.Auxiliary.LocalZipMapFileProvider")
lean_config_manager.set_property(
"factor-file-provider",
"QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider")
if data_purchase_limit is not None:
config = lean_config_manager.get_lean_config()
if config.get(
"data-provider",
None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider":
container.logger().warn(
"--data-purchase-limit is ignored because the data provider is not set to download from the API, use --download-data to set that up"
)
data_purchase_limit = None
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, debugging_method, data_purchase_limit)
lean_runner = container.lean_runner()
lean_runner.run_lean(lean_config, "backtesting", algorithm_file, output,
engine_image, debugging_method)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"commands": [],
"environment": {},
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_output": lambda chunk: _check_docker_output(chunk, port)
}
lean_runner = container.lean_runner()
if project_config.get("algorithm-language", "Python") == "Python":
lean_runner.set_up_python_options(project, "/Lean/Launcher/bin/Debug/Notebooks", run_options)
else:
lean_runner.set_up_csharp_options(project, run_options)
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and "port is already allocated" in msg:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
def optimize(project: Path, output: Optional[Path], detach: bool,
optimizer_config: Optional[Path], strategy: Optional[str],
target: Optional[str], target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
release: bool, image: Optional[str], update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
By default an interactive wizard is shown letting you configure the optimizer.
If --optimizer-config or --strategy is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
If --strategy is given the optimizer configuration is read from the given options.
In this case --strategy, --target, --target-direction and --parameter become required.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
optimizer_config_manager = container.optimizer_config_manager()
config = None
if optimizer_config is not None and strategy is not None:
raise RuntimeError(
"--optimizer-config and --strategy are mutually exclusive")
if optimizer_config is not None:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
elif strategy is not None:
ensure_options(["strategy", "target", "target_direction", "parameter"])
optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
else:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/optimization/parameters")
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
if config is None:
# noinspection PyUnboundLocalVariable
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_optimization_id(output))
lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging"
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None, release, detach)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
project_manager.copy_code(algorithm_file.parent, output / "code")
success = container.docker_manager().run_image(engine_image, **run_options)
logger = container.logger()
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(f"The output will be stored in '{relative_output_dir}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
elif success:
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
def optimize(project: Path, output: Optional[Path],
optimizer_config: Optional[Path], image: Optional[str],
update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
When --optimizer-config is not set, an interactive prompt will be shown to configure the optimizer.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
if optimizer_config is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/tutorials/optimization/project-parameters"
)
optimizer_config_manager = container.optimizer_config_manager()
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
else:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None, None)
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if success:
logger = container.logger()
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def build(root: Path, tag: str) -> None:
"""Build Docker images of your own version of LEAN and the Alpha Streams SDK.
\b
ROOT must point to a directory containing the LEAN repository and the Alpha Streams SDK repository:
https://github.com/QuantConnect/Lean & https://github.com/QuantConnect/AlphaStreams
When ROOT is not given, the current directory is used as root directory.
\b
This command performs the following actions:
1. The lean-cli/foundation:latest image is built from Lean/DockerfileLeanFoundation(ARM).
2. LEAN is compiled in a Docker container using the lean-cli/foundation:latest image.
3. The Alpha Streams SDK is compiled in a Docker container using the lean-cli/foundation:latest image.
4. The lean-cli/engine:latest image is built from Lean/Dockerfile using lean-cli/foundation:latest as base image.
5. The lean-cli/research:latest image is built from Lean/DockerfileJupyter using lean-cli/engine:latest as base image.
6. The default engine image is set to lean-cli/engine:latest.
7. The default research image is set to lean-cli/research:latest.
When the foundation Dockerfile is the same as the official foundation Dockerfile,
quantconnect/lean:foundation is used instead of building a custom foundation image.
"""
lean_dir = root / "Lean"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/Lean to '{lean_dir}'"
)
alpha_streams_dir = root / "AlphaStreams"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/AlphaStreams to '{alpha_streams_dir}'"
)
(root / "DataLibraries").mkdir(exist_ok=True)
if container.platform_manager().is_host_arm():
foundation_dockerfile = lean_dir / "DockerfileLeanFoundationARM"
else:
foundation_dockerfile = lean_dir / "DockerfileLeanFoundation"
if _is_foundation_dockerfile_same_as_cloud(foundation_dockerfile):
foundation_image = DockerImage(name="quantconnect/lean",
tag="foundation")
container.docker_manager().pull_image(foundation_image)
else:
foundation_image = DockerImage(name="lean-cli/foundation", tag=tag)
_build_image(root, foundation_dockerfile, None, foundation_image)
_compile_csharp(root, lean_dir, foundation_image)
_compile_csharp(root, alpha_streams_dir, foundation_image)
custom_engine_image = DockerImage(name="lean-cli/engine", tag=tag)
_build_image(root, lean_dir / "Dockerfile", foundation_image,
custom_engine_image)
custom_research_image = DockerImage(name="lean-cli/research", tag=tag)
_build_image(root, lean_dir / "DockerfileJupyter", custom_engine_image,
custom_research_image)
logger = container.logger()
cli_config_manager = container.cli_config_manager()
logger.info(f"Setting default engine image to '{custom_engine_image}'")
cli_config_manager.engine_image.set_value(str(custom_engine_image))
logger.info(f"Setting default research image to '{custom_research_image}'")
cli_config_manager.research_image.set_value(str(custom_research_image))
