def _save_properties(self, lean_config: Dict[str, Any],
properties: List[str]) -> None:
"""Persistently save properties in the Lean configuration.
:param lean_config: the dict containing all properties
:param properties: the names of the properties to save persistently
"""
from lean.container import container
container.lean_config_manager().set_properties(
{key: lean_config[key]
for key in properties})
def live(project: Path, environment: str, output: Optional[Path],
image: Optional[str], update: bool) -> None:
"""Start live trading a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
ENVIRONMENT must be the name of an environment in the Lean configuration file with live-mode set to true.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
if output is None:
output = algorithm_file.parent / "live" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
environment, algorithm_file, None)
if "environments" not in lean_config or environment not in lean_config[
"environments"]:
lean_config_path = lean_config_manager.get_lean_config_path()
raise MoreInfoError(
f"{lean_config_path} does not contain an environment named '{environment}'",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
if not lean_config["environments"][environment]["live-mode"]:
raise MoreInfoError(
f"The '{environment}' is not a live trading environment (live-mode is set to false)",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
_raise_for_missing_properties(lean_config, environment,
lean_config_manager.get_lean_config_path())
_start_iqconnect_if_necessary(lean_config, environment)
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_runner = container.lean_runner()
lean_runner.run_lean(environment, algorithm_file, output, engine_image,
None)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def create_project(name: str, language: str) -> None:
"""Create a new project containing starter code.
If NAME is a path containing subdirectories those will be created automatically.
The default language can be set using `lean config set default-language python/csharp`.
"""
cli_config_manager = container.cli_config_manager()
language = language if language is not None else cli_config_manager.default_language.get_value()
if language is None:
raise MoreInfoError(
"Please specify a language with --language or set the default language using `lean config set default-language python/csharp`",
"https://www.lean.io/docs/lean-cli/projects/project-management")
full_path = Path.cwd() / name
if not container.path_manager().is_path_valid(full_path):
raise MoreInfoError(f"'{name}' is not a valid path",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors")
is_library_project = False
try:
library_dir = container.lean_config_manager().get_cli_root_directory() / "Library"
is_library_project = library_dir in full_path.parents
except:
# get_cli_root_directory() raises an error if there is no such directory
pass
if is_library_project and language == "python" and not full_path.name.isidentifier():
raise RuntimeError(
f"'{full_path.name}' is not a valid Python identifier, which is required for Python library projects to be importable")
if full_path.exists():
raise RuntimeError(f"A project named '{name}' already exists, please choose a different name")
else:
project_manager = container.project_manager()
project_manager.create_new_project(full_path, QCLanguage.Python if language == "python" else QCLanguage.CSharp)
# Convert the project name into a valid class name by removing all non-alphanumeric characters
class_name = re.sub(f"[^a-zA-Z0-9]", "", "".join(map(_capitalize, full_path.name.split(" "))))
if language == "python":
main_name = "main.py"
main_content = DEFAULT_PYTHON_MAIN if not is_library_project else LIBRARY_PYTHON_MAIN
else:
main_name = "Main.cs"
main_content = DEFAULT_CSHARP_MAIN if not is_library_project else LIBRARY_CSHARP_MAIN
with (full_path / main_name).open("w+", encoding="utf-8") as file:
file.write(main_content.replace("$CLASS_NAME$", class_name).replace("$PROJECT_NAME$", full_path.name))
with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_NOTEBOOK if language == "python" else DEFAULT_CSHARP_NOTEBOOK)
logger = container.logger()
logger.info(f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'")
def _save_properties(cls, lean_config: Dict[str, Any],
properties: List[str]) -> None:
"""Persistently save properties in the Lean configuration.
:param lean_config: the dict containing all properties
:param properties: the names of the properties to save persistently
"""
lean_config_manager = container.lean_config_manager()
for prop in properties:
lean_config_manager.set_property(prop, lean_config[prop])
def invoke(self, ctx):
if self._requires_lean_config:
try:
# This method will throw if the directory cannot be found
container.lean_config_manager().get_cli_root_directory()
except Exception:
# Abort with a display-friendly error message if the command requires a Lean config
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/user-guides/troubleshooting#02-Common-errors")
if self._requires_docker and "pytest" not in sys.modules:
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if platform.system() == "Linux":
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if platform.system() == "Linux" and os.getuid() != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password")
args = ["sudo", "--preserve-env=HOME", sys.executable, *sys.argv]
os.execlp(args[0], *args)
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def _get_project_directories() -> List[Path]:
directories_to_check = [
container.lean_config_manager().get_cli_root_directory()
]
project_directories = []
while len(directories_to_check) > 0:
directory = directories_to_check.pop(0)
config_file = directory / PROJECT_CONFIG_FILE_NAME
if config_file.is_file():
project_directories.append(directory)
else:
directories_to_check.extend(d for d in directory.iterdir()
if d.is_dir())
return project_directories
def _migrate_python_vscode(project_dir: Path) -> None:
launch_json_path = project_dir / ".vscode" / "launch.json"
if not launch_json_path.is_file():
return
current_content = json.loads(launch_json_path.read_text(encoding="utf-8"))
if "configurations" not in current_content or not isinstance(
current_content["configurations"], list):
return
config = next((c for c in current_content["configurations"]
if c["name"] == "Debug with Lean CLI"), None)
if config is None:
return
made_changes = False
has_library_mapping = False
library_dir = container.lean_config_manager().get_cli_root_directory(
) / "Library"
if not library_dir.is_dir():
library_dir = None
for mapping in config["pathMappings"]:
if mapping["localRoot"] == "${workspaceFolder}" and mapping[
"remoteRoot"] == "/Lean/Launcher/bin/Debug":
mapping["remoteRoot"] = "/LeanCLI"
made_changes = True
if library_dir is not None and mapping["localRoot"] == str(
library_dir) and mapping["remoteRoot"] == "/Library":
has_library_mapping = True
if library_dir is not None and not has_library_mapping:
config["pathMappings"].append({
"localRoot": str(library_dir),
"remoteRoot": "/Library"
})
made_changes = True
if made_changes:
launch_json_path.write_text(json.dumps(current_content, indent=4),
encoding="utf-8")
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options = {
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
},
str(project): {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_run": lambda: webbrowser.open(f"http://localhost:{port}/")
}
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
if "port is already allocated" in error.explanation:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
def _parse_config_option(self, ctx: click.Context, param: click.Parameter, value: Optional[Path]) -> None:
"""Parses the --config option."""
if value is not None:
lean_config_manager = container.lean_config_manager()
lean_config_manager.set_default_lean_config_path(value)
def init() -> None:
"""Bootstrap a Lean CLI project."""
current_dir = Path.cwd()
data_dir = current_dir / Config.default_data_directory_name
lean_config_path = current_dir / Config.default_lean_config_file_name
# Abort if one of the files we are going to create already exists to prevent us from overriding existing files
for path in [data_dir, lean_config_path]:
if path.exists():
relative_path = path.relative_to(current_dir)
raise RuntimeError(
f"{relative_path} already exists, please run this command in an empty directory"
)
logger = container.logger()
# Warn the user if the current directory is not empty
if next(current_dir.iterdir(), None) is not None:
logger.info(
"This command will bootstrap a Lean CLI project in the current directory"
)
click.confirm("The current directory is not empty, continue?",
default=False,
abort=True)
# Download the Lean repository
logger.info("Downloading latest sample data from the Lean repository...")
tmp_directory = Path(tempfile.mkdtemp())
# We download the entire Lean repository and extract the data and the launcher's config file
# GitHub doesn't allow downloading a specific directory
# Since we need ~80% of the total repository in terms of file size this shouldn't be too big of a problem
with requests.get(
"https://github.com/QuantConnect/Lean/archive/master.zip",
stream=True) as response:
response.raise_for_status()
with (tmp_directory / "master.zip").open("wb") as file:
for chunk in response.iter_content(chunk_size=8192):
file.write(chunk)
# Extract the downloaded repository
with zipfile.ZipFile(tmp_directory / "master.zip") as zip_file:
zip_file.extractall(tmp_directory / "master")
# Copy the data directory
shutil.copytree(tmp_directory / "master" / "Lean-master" / "Data",
data_dir)
# Create the config file
lean_config_manager = container.lean_config_manager()
config = (tmp_directory / "master" / "Lean-master" / "Launcher" /
"config.json").read_text()
config = lean_config_manager.clean_lean_config(config)
# Update the data-folder configuration
config = config.replace(
'"data-folder": "../../../Data/"',
f'"data-folder": "{Config.default_data_directory_name}"')
with lean_config_path.open("w+") as file:
file.write(config)
# Create files which make debugging and autocompletion possible
extra_files = {
"LeanCLI.csproj": CSPROJ,
".idea/workspace.xml": IDEA_WORKSPACE_XML,
".vscode/launch.json": VSCODE_LAUNCH_JSON
}
for location, content in extra_files.items():
path = Path(Path.cwd() / location)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w+") as file:
file.write(content)
# Prompt for some general configuration if not set yet
cli_config_manager = container.cli_config_manager()
if cli_config_manager.default_language.get_value() is None:
default_language = click.prompt(
"What should the default language for new projects be?",
type=click.Choice(
cli_config_manager.default_language.allowed_values))
cli_config_manager.default_language.set_value(default_language)
logger.info(f"""
Successfully bootstrapped your Lean CLI project!
The following structure has been created:
- {Config.default_lean_config_file_name} contains the configuration used when running the LEAN engine locally
- {Config.default_data_directory_name}/ contains the data that is used when running the LEAN engine locally
Here are some commands to get you going:
- Run `lean create-project "My Project"` to create a new project with starter code
- Run `lean backtest "My Project"` to backtest a project locally with the data in {Config.default_data_directory_name}/
""".strip())
def report(backtest_data_source_file: Path,
live_data_source_file: Optional[Path],
report_destination: Path,
strategy_name: Optional[str],
strategy_version: Optional[str],
strategy_description: Optional[str],
overwrite: bool,
image: Optional[str],
update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it")
project_directory = _find_project_directory(backtest_data_source_file)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder": "/Lean/Data",
"strategy-name": strategy_name or "",
"strategy-version": strategy_version or "",
"strategy-description": strategy_description or "",
"live-data-source-file": "live-data-source-file.json" if live_data_source_file is not None else "",
"backtest-data-source-file": "backtest-data-source-file.json",
"report-destination": "/Results/report.html",
"environment": "report",
"log-handler": "QuantConnect.Logging.CompositeLogHandler",
"messaging-handler": "QuantConnect.Messaging.Messaging",
"job-queue-handler": "QuantConnect.Queues.JobQueue",
"api-handler": "QuantConnect.Api.Api",
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler": "QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider": "DataChannelProvider",
"environments": {
"report": {
"live-mode": False,
"setup-handler": "QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler": "QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler": "QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider": "QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
output_dir = container.temp_manager().create_temporary_directory()
config_path = output_dir / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"working_dir": "/Lean/Report/bin/Debug",
"entrypoint": ["dotnet", "QuantConnect.Report.dll"],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_data_source_file),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "ro"
},
str(output_dir): {
"bind": "/Results",
"mode": "rw"
}
}
}
if live_data_source_file is not None:
run_options["mounts"].append(Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_data_source_file),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information")
report_destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output_dir / "report.html", report_destination)
logger = container.logger()
logger.info(f"Successfully generated report to '{report_destination}'")
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def backtest(project: Path, output: Optional[Path], debug: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
image: Optional[str], update: bool) -> None:
"""Backtest a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
Go to the following url to learn how to debug backtests locally using the Lean CLI:
https://www.lean.io/docs/lean-cli/tutorials/backtesting/debugging-local-backtests
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
lean_config_manager = container.lean_config_manager()
if output is None:
output = algorithm_file.parent / "backtests" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
debugging_method = None
if debug == "pycharm":
debugging_method = DebuggingMethod.PyCharm
_migrate_dotnet_5_python_pycharm(algorithm_file.parent)
elif debug == "ptvsd":
debugging_method = DebuggingMethod.PTVSD
_migrate_dotnet_5_python_vscode(algorithm_file.parent)
elif debug == "vsdbg":
debugging_method = DebuggingMethod.VSDBG
_migrate_dotnet_5_csharp_vscode(algorithm_file.parent)
elif debug == "rider":
debugging_method = DebuggingMethod.Rider
_migrate_dotnet_5_csharp_rider(algorithm_file.parent)
if download_data:
organization = _select_organization()
lean_config_manager.set_property("job-organization-id",
organization.id)
lean_config_manager.set_property(
"data-provider",
"QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider")
lean_config_manager.set_property(
"map-file-provider",
"QuantConnect.Data.Auxiliary.LocalZipMapFileProvider")
lean_config_manager.set_property(
"factor-file-provider",
"QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider")
if data_purchase_limit is not None:
config = lean_config_manager.get_lean_config()
if config.get(
"data-provider",
None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider":
container.logger().warn(
"--data-purchase-limit is ignored because the data provider is not set to download from the API, use --download-data to set that up"
)
data_purchase_limit = None
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, debugging_method, data_purchase_limit)
lean_runner = container.lean_runner()
lean_runner.run_lean(lean_config, "backtesting", algorithm_file, output,
engine_image, debugging_method)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def init() -> None:
"""Scaffold a Lean configuration file and data directory."""
current_dir = Path.cwd()
data_dir = current_dir / DEFAULT_DATA_DIRECTORY_NAME
lean_config_path = current_dir / DEFAULT_LEAN_CONFIG_FILE_NAME
# Abort if one of the files we are going to create already exists to prevent us from overriding existing files
for path in [data_dir, lean_config_path]:
if path.exists():
relative_path = path.relative_to(current_dir)
raise MoreInfoError(f"{relative_path} already exists, please run this command in an empty directory",
"https://www.lean.io/docs/lean-cli/initialization/directory-structure#02-lean-init")
logger = container.logger()
# Warn the user if the current directory is not empty
if next(current_dir.iterdir(), None) is not None:
logger.info("This command will create a Lean configuration file and data directory in the current directory")
click.confirm("The current directory is not empty, continue?", default=False, abort=True)
# Download the Lean repository
tmp_directory = container.temp_manager().create_temporary_directory()
_download_repository(tmp_directory / "master.zip")
# Extract the downloaded repository
with zipfile.ZipFile(tmp_directory / "master.zip") as zip_file:
zip_file.extractall(tmp_directory / "master")
# Copy the data directory
shutil.copytree(tmp_directory / "master" / "Lean-master" / "Data", data_dir)
# Create the config file
lean_config_manager = container.lean_config_manager()
config = (tmp_directory / "master" / "Lean-master" / "Launcher" / "config.json").read_text(encoding="utf-8")
config = lean_config_manager.clean_lean_config(config)
lean_config_manager.store_known_lean_config_path(lean_config_path)
# Update the data-folder configuration
config = config.replace('"data-folder": "../../../Data/"', f'"data-folder": "{DEFAULT_DATA_DIRECTORY_NAME}"')
with lean_config_path.open("w+", encoding="utf-8") as file:
file.write(config)
# Prompt for some general configuration if not set yet
cli_config_manager = container.cli_config_manager()
if cli_config_manager.default_language.get_value() is None:
default_language = click.prompt("What should the default language for new projects be?",
type=click.Choice(cli_config_manager.default_language.allowed_values))
cli_config_manager.default_language.set_value(default_language)
logger.info(f"""
The following objects have been created:
- {DEFAULT_LEAN_CONFIG_FILE_NAME} contains the configuration used when running the LEAN engine locally
- {DEFAULT_DATA_DIRECTORY_NAME}/ contains the data that is used when running the LEAN engine locally
The following documentation pages may be useful:
- Setting up local autocomplete: https://www.lean.io/docs/lean-cli/projects/autocomplete
- Synchronizing projects with the cloud: https://www.lean.io/docs/lean-cli/projects/cloud-synchronization
Here are some commands to get you going:
- Run `lean create-project "My Project"` to create a new project with starter code
- Run `lean cloud pull` to download all your QuantConnect projects to your local drive
- Run `lean backtest "My Project"` to backtest a project locally with the data in {DEFAULT_DATA_DIRECTORY_NAME}/
""".strip())
# Prompt to create a desktop shortcut for the local GUI if the user is in an organization with a subscription
api_client = container.api_client()
if api_client.is_authenticated():
for simple_organization in api_client.organizations.get_all():
organization = api_client.organizations.get(simple_organization.id)
modules_product = next((p for p in organization.products if p.name == "Modules"), None)
if modules_product is None:
continue
if any(i for i in modules_product.items if i.productId in GUI_PRODUCT_SUBSCRIPTION_IDS):
container.shortcut_manager().prompt_if_necessary(simple_organization.id)
break
def start(organization: Optional[str], port: int, no_open: bool,
shortcut: bool, gui: Optional[Path], shortcut_launch: bool) -> None:
"""Start the local GUI."""
logger = container.logger()
docker_manager = container.docker_manager()
temp_manager = container.temp_manager()
module_manager = container.module_manager()
api_client = container.api_client()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is not None:
if gui_container.status == "running":
if shortcut_launch:
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
webbrowser.open(url)
return
else:
_error(
"The local GUI is already running, run `lean gui restart` to restart it or `lean gui stop` to stop it",
shortcut_launch)
gui_container.remove()
if organization is not None:
organization_id = _get_organization_id(organization, shortcut_launch)
else:
organizations = api_client.organizations.get_all()
options = [
Option(id=organization.id, label=organization.name)
for organization in organizations
]
organization_id = logger.prompt_list(
"Select the organization with the local GUI module subscription",
options)
module_manager.install_module(GUI_PRODUCT_INSTALL_ID, organization_id)
shortcut_manager = container.shortcut_manager()
if shortcut:
shortcut_manager.create_shortcut(organization_id)
else:
shortcut_manager.prompt_if_necessary(organization_id)
# The dict containing all options passed to `docker run`
# See all available options at https://docker-py.readthedocs.io/en/stable/containers.html
run_options: Dict[str, Any] = {
"name": LOCAL_GUI_CONTAINER_NAME,
"detach": True,
"remove": False,
"commands": [],
"environment": {
"PYTHONUNBUFFERED": "1",
"QC_LOCAL_GUI": "true",
"QC_DOCKER_HOST_SYSTEM": platform.system(),
"QC_DOCKER_HOST_MACHINE": platform.machine(),
"QC_ORGANIZATION_ID": organization_id,
"QC_API": os.environ.get("QC_API", "")
},
"mounts": [],
"volumes": {},
"ports": {
"5612": str(port)
}
}
# Cache the site-packages so we don't re-install everything when the container is restarted
docker_manager.create_volume("lean_cli_gui_python")
run_options["volumes"]["lean_cli_gui_python"] = {
"bind": "/root/.local/lib/python3.9/site-packages",
"mode": "rw"
}
# Update PATH in the GUI container to add executables installed with pip
run_options["commands"].append('export PATH="$PATH:/root/.local/bin"')
package_file_name = module_manager.get_installed_packages_by_module(
GUI_PRODUCT_INSTALL_ID)[0].get_file_name()
with zipfile.ZipFile(Path.home() / ".lean" / "modules" /
package_file_name) as package_file:
content_file_names = [
f.replace("content/", "") for f in package_file.namelist()
if f.startswith("content/")
]
wheel_file_name = next(f for f in content_file_names
if f.endswith(".whl"))
terminal_file_name = next(f for f in content_file_names
if f.endswith(".zip"))
# Install the CLI in the GUI container
run_options["commands"].append("pip uninstall -y lean")
if lean.__version__ == "dev":
lean_cli_dir = str(
Path(__file__).absolute().parent.parent.parent.parent)
logger.info(
f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli"
)
run_options["volumes"][str(lean_cli_dir)] = {
"bind": "/lean-cli",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
else:
run_options["commands"].append(
"pip install --user --progress-bar off --upgrade lean")
# Install the GUI in the GUI container
run_options["commands"].append("pip uninstall -y leangui")
if gui is None:
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{wheel_file_name} > /{wheel_file_name}"
)
run_options["commands"].append(
f"pip install --user --progress-bar off /{wheel_file_name}")
elif gui.is_file():
run_options["mounts"].append(
Mount(target=f"/{gui.name}",
source=str(gui),
type="bind",
read_only=True))
run_options["commands"].append(
f"pip install --user --progress-bar off /{gui.name}")
else:
run_options["volumes"][str(gui)] = {
"bind": "/lean-cli-gui",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli-gui")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
# Extract the terminal in the GUI container
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{terminal_file_name} > /{terminal_file_name}"
)
run_options["commands"].append(
f"unzip -o /{terminal_file_name} -d /terminal")
# Write correct streaming url to /terminal/local.socket.host.conf
run_options["commands"].append(
f'echo "ws://localhost:{port}/streaming" > /terminal/local.socket.host.conf'
)
# Mount the `lean init` directory in the GUI container
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
run_options["volumes"][str(cli_root_dir)] = {
"bind": "/LeanCLI",
"mode": "rw"
}
# Mount the global config directory in the GUI container
run_options["volumes"][str(Path("~/.lean").expanduser())] = {
"bind": "/root/.lean",
"mode": "rw"
}
# Mount a directory to the tmp directory in the GUI container
gui_tmp_directory = temp_manager.create_temporary_directory()
run_options["volumes"][str(gui_tmp_directory)] = {
"bind": "/tmp",
"mode": "rw"
}
# Set up the path mappings between paths in the host system and paths in the GUI container
run_options["environment"]["DOCKER_PATH_MAPPINGS"] = json.dumps({
"/LeanCLI":
cli_root_dir.as_posix(),
"/root/.lean":
Path("~/.lean").expanduser().as_posix(),
"/tmp":
gui_tmp_directory.as_posix()
})
# Mount the Docker socket in the GUI container
run_options["mounts"].append(
Mount(target="/var/run/docker.sock",
source="/var/run/docker.sock",
type="bind",
read_only=False))
# Run the GUI in the GUI container
run_options["commands"].append("cd /LeanCLI")
run_options["commands"].append(f"leangui")
# Don't delete temporary directories when the command exits, the container will still need them
temp_manager.delete_temporary_directories_when_done = False
logger.info("Starting the local GUI, this may take some time...")
# Pull the Docker images used by the local GUI
# If this is done while the local GUI is running there is a big delay between pressing Backtest and seeing it run
update_manager = container.update_manager()
cli_config_manager = container.cli_config_manager()
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_engine_image(), False)
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_research_image(), False)
try:
docker_manager.run_image(
DockerImage(name="python", tag="3.9.6-buster"), **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
_error(
f"Port {port} is already in use, please specify a different port using --port <number>",
shortcut_launch)
raise error
url = f"http://localhost:{port}/"
# Wait until the GUI is running
while True:
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
if shortcut_launch:
_error(
"Something went wrong while starting the local GUI, run `lean gui logs` for more information",
shortcut_launch)
else:
_error(
"Something went wrong while starting the local GUI, see the logs above for more information",
shortcut_launch)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
logger.info(f"The local GUI has started and is running on {url}")
if not no_open:
webbrowser.open(url)
def generate(start: datetime, end: datetime, symbol_count: int,
security_type: str, resolution: str, data_density: str,
include_coarse: bool, market: str, image: Optional[str],
update: bool) -> None:
"""Generate random market data.
\b
This uses the random data generator in LEAN to generate realistic market data using a Brownian motion model.
This generator supports the following security types, tick types and resolutions:
| Security type | Generated tick types | Supported resolutions |
| ------------- | -------------------- | ------------------------------------ |
| Equity | Trade | Tick, Second, Minute, Hour and Daily |
| Forex | Quote | Tick, Second, Minute, Hour and Daily |
| CFD | Quote | Tick, Second, Minute, Hour and Daily |
| Future | Trade and Quote | Tick, Second, Minute, Hour and Daily |
| Crypto | Trade and Quote | Tick, Second, Minute, Hour and Daily |
| Option | Trade and Quote | Minute |
\b
The following data densities are available:
- Dense: at least one data point per resolution step.
- Sparse: at least one data point per 5 resolution steps.
- VerySparse: at least one data point per 50 resolution steps.
\b
Example which generates minute data for 100 equity symbols since 2015-01-01:
$ lean data generate --start=20150101 --symbol-count=100
\b
Example which generates daily data for 100 crypto symbols since 2015-01-01:
$ lean data generate --start=20150101 --symbol-count=100 --security-type=Crypto --resolution=Daily
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options = {
"entrypoint": [
"dotnet", "QuantConnect.ToolBox.dll", "--destination-dir",
"/Lean/Data", "--app", "randomdatagenerator", "--start",
start.strftime("%Y%m%d"), "--end",
end.strftime("%Y%m%d"), "--symbol-count",
str(symbol_count), "--security-type", security_type,
"--resolution", resolution, "--data-density", data_density,
"--include-coarse",
str(include_coarse).lower(), "--market",
market.lower()
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
}
}
}
engine_image = container.cli_config_manager().get_engine_image(image)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the random data generator, see the logs above for more information"
)
def invoke(self, ctx: click.Context):
if self._requires_lean_config:
lean_config_manager = container.lean_config_manager()
try:
# This method will raise an error if the directory cannot be found
lean_config_manager.get_cli_root_directory()
except Exception:
# Use one of the cached Lean config locations to avoid having to abort the command
lean_config_paths = lean_config_manager.get_known_lean_config_paths(
)
if len(lean_config_paths) > 0:
lean_config_path = container.logger().prompt_list(
"Select the Lean configuration file to use", [
Option(id=p, label=str(p))
for p in lean_config_paths
])
lean_config_manager.set_default_lean_config_path(
lean_config_path)
else:
# Abort with a display-friendly error message if the command requires a Lean config and none found
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors"
)
if self._requires_docker and "pytest" not in sys.modules:
is_system_linux = container.platform_manager().is_system_linux()
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if is_system_linux:
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if is_system_linux and os.getuid(
) != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password"
)
args = [
"sudo", "--preserve-env=HOME", sys.executable, *sys.argv
]
os.execlp(args[0], *args)
if self._allow_unknown_options:
# Unknown options are passed to ctx.args and need to be parsed manually
# We parse them to ctx.params so they're available like normal options
# Because of this all commands with allow_unknown_options=True must have a **kwargs argument
arguments = list(
itertools.chain(*[arg.split("=") for arg in ctx.args]))
skip_next = False
for index in range(len(arguments) - 1):
if skip_next:
skip_next = False
continue
if arguments[index].startswith("--"):
option = arguments[index].replace("--", "")
value = arguments[index + 1]
ctx.params[option] = value
skip_next = True
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def _migrate_python_pycharm(project_dir: Path) -> None:
workspace_xml_path = project_dir / ".idea" / "workspace.xml"
if not workspace_xml_path.is_file():
return
xml_manager = container.xml_manager()
current_content = xml_manager.parse(
workspace_xml_path.read_text(encoding="utf-8"))
config = current_content.find(
'.//configuration[@name="Debug with Lean CLI"]')
if config is None:
return
path_mappings = config.find(
'.//PathMappingSettings/option[@name="pathMappings"]/list')
if path_mappings is None:
return
made_changes = False
has_library_mapping = False
library_dir = container.lean_config_manager().get_cli_root_directory(
) / "Library"
if library_dir.is_dir():
library_dir = f"$PROJECT_DIR$/{os.path.relpath(library_dir, project_dir)}".replace(
"\\", "/")
else:
library_dir = None
for mapping in path_mappings.findall(".//mapping"):
if mapping.get("local-root") == "$PROJECT_DIR$" and mapping.get(
"remote-root") == "/Lean/Launcher/bin/Debug":
mapping.set("remote-root", "/LeanCLI")
made_changes = True
if library_dir is not None \
and mapping.get("local-root") == library_dir \
and mapping.get("remote-root") == "/Library":
has_library_mapping = True
if library_dir is not None and not has_library_mapping:
library_mapping = xml_manager.parse("<mapping/>")
library_mapping.set("local-root", library_dir)
library_mapping.set("remote-root", "/Library")
path_mappings.append(library_mapping)
made_changes = True
if made_changes:
workspace_xml_path.write_text(xml_manager.to_string(current_content),
encoding="utf-8")
logger = container.logger()
logger.warn(
"Your run configuration has been updated to work with the latest version of LEAN"
)
logger.warn(
"Please restart the debugger in PyCharm and run this command again"
)
raise click.Abort()
def backtest(project: Path, output: Optional[Path], detach: bool,
debug: Optional[str], data_provider: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
release: bool, image: Optional[str], update: bool) -> None:
"""Backtest a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
Go to the following url to learn how to debug backtests locally using the Lean CLI:
https://www.lean.io/docs/lean-cli/tutorials/backtesting/debugging
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
lean_config_manager = container.lean_config_manager()
if output is None:
output = algorithm_file.parent / "backtests" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
debugging_method = None
if debug == "pycharm":
debugging_method = DebuggingMethod.PyCharm
_migrate_python_pycharm(algorithm_file.parent)
elif debug == "ptvsd":
debugging_method = DebuggingMethod.PTVSD
_migrate_python_vscode(algorithm_file.parent)
elif debug == "vsdbg":
debugging_method = DebuggingMethod.VSDBG
_migrate_csharp_vscode(algorithm_file.parent)
elif debug == "rider":
debugging_method = DebuggingMethod.Rider
_migrate_csharp_rider(algorithm_file.parent)
if debugging_method is not None and detach:
raise RuntimeError(
"Running a debugging session in a detached container is not supported"
)
if algorithm_file.name.endswith(".cs"):
_migrate_csharp_csproj(algorithm_file.parent)
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, debugging_method)
if download_data:
data_provider = QuantConnectDataProvider.get_name()
if data_provider is not None:
data_provider = next(dp for dp in all_data_providers
if dp.get_name() == data_provider)
data_provider.build(lean_config, container.logger()).configure(
lean_config, "backtesting")
lean_config_manager.configure_data_purchase_limit(lean_config,
data_purchase_limit)
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_backtest_id(output))
lean_runner = container.lean_runner()
lean_runner.run_lean(lean_config, "backtesting", algorithm_file, output,
engine_image, debugging_method, release, detach)
def research(project: Path, port: int, data_provider: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
detach: bool, no_open: bool, image: Optional[str],
update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
lean_config["composer-dll-directory"] = "/Lean/Launcher/bin/Debug"
if download_data:
data_provider = QuantConnectDataProvider.get_name()
if data_provider is not None:
data_provider = next(dp for dp in all_data_providers
if dp.get_name() == data_provider)
data_provider.build(lean_config, container.logger()).configure(
lean_config, "backtesting")
lean_config_manager.configure_data_purchase_limit(lean_config,
data_purchase_limit)
lean_runner = container.lean_runner()
temp_manager = container.temp_manager()
run_options = lean_runner.get_basic_docker_config(
lean_config, algorithm_file, temp_manager.create_temporary_directory(),
None, False, detach)
# Mount the config in the notebooks directory as well
local_config_path = next(m["Source"] for m in run_options["mounts"]
if m["Target"].endswith("config.json"))
run_options["mounts"].append(
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(local_config_path),
type="bind",
read_only=True))
# Jupyter Lab runs on port 8888, we expose it to the local port specified by the user
run_options["ports"]["8888"] = str(port)
# Open the browser as soon as Jupyter Lab has started
if detach or not no_open:
run_options["on_output"] = lambda chunk: _check_docker_output(
chunk, port)
# Give container an identifiable name when running it from the GUI
if container.module_manager().is_module_installed(GUI_PRODUCT_INSTALL_ID):
project_id = container.project_config_manager().get_local_id(
algorithm_file.parent)
run_options["name"] = f"lean_cli_gui_research_{project_id}"
# Make Ctrl+C stop Jupyter Lab immediately
run_options["stop_signal"] = "SIGKILL"
# Mount the project to the notebooks directory
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Allow notebooks to be embedded in iframes
run_options["commands"].append("mkdir -p ~/.jupyter")
run_options["commands"].append(
'echo "c.NotebookApp.disable_check_xsrf = True\nc.NotebookApp.tornado_settings = {\'headers\': {\'Content-Security-Policy\': \'frame-ancestors self *\'}}" > ~/.jupyter/jupyter_notebook_config.py'
)
# Hide headers in notebooks
run_options["commands"].append(
"mkdir -p ~/.ipython/profile_default/static/custom")
run_options["commands"].append(
'echo "#header-container { display: none !important; }" > ~/.ipython/profile_default/static/custom/custom.css'
)
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
research_image = cli_config_manager.get_research_image(
image or project_config.get("research-image", None))
container.update_manager().pull_docker_image_if_necessary(
research_image, update)
try:
container.docker_manager().run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
raise RuntimeError(
f"Port {port} is already in use, please specify a different port using --port <number>"
)
raise error
if detach:
temp_manager.delete_temporary_directories_when_done = False
logger = container.logger()
relative_project_dir = algorithm_file.parent.relative_to(
lean_config_manager.get_cli_root_directory())
logger.info(
f"Successfully started Jupyter Lab environment for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(
"You can use Docker's own commands to manage the detached container"
)
def optimize(project: Path, output: Optional[Path],
optimizer_config: Optional[Path], image: Optional[str],
update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
When --optimizer-config is not set, an interactive prompt will be shown to configure the optimizer.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
if optimizer_config is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/tutorials/optimization/project-parameters"
)
optimizer_config_manager = container.optimizer_config_manager()
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
else:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None, None)
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if success:
logger = container.logger()
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def optimize(project: Path, output: Optional[Path], detach: bool,
optimizer_config: Optional[Path], strategy: Optional[str],
target: Optional[str], target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
release: bool, image: Optional[str], update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
By default an interactive wizard is shown letting you configure the optimizer.
If --optimizer-config or --strategy is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
If --strategy is given the optimizer configuration is read from the given options.
In this case --strategy, --target, --target-direction and --parameter become required.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
optimizer_config_manager = container.optimizer_config_manager()
config = None
if optimizer_config is not None and strategy is not None:
raise RuntimeError(
"--optimizer-config and --strategy are mutually exclusive")
if optimizer_config is not None:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
elif strategy is not None:
ensure_options(["strategy", "target", "target_direction", "parameter"])
optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
else:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/optimization/parameters")
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
if config is None:
# noinspection PyUnboundLocalVariable
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_optimization_id(output))
lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging"
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None, release, detach)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
project_manager.copy_code(algorithm_file.parent, output / "code")
success = container.docker_manager().run_image(engine_image, **run_options)
logger = container.logger()
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(f"The output will be stored in '{relative_output_dir}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
elif success:
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
def report(backtest_results: Optional[Path], live_results: Optional[Path],
report_destination: Path, detach: bool,
strategy_name: Optional[str], strategy_version: Optional[str],
strategy_description: Optional[str], overwrite: bool,
image: Optional[str], update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
If --backtest-results is not given, a report is generated for the most recent local backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(
f"{report_destination} already exists, use --overwrite to overwrite it"
)
if backtest_results is None:
backtest_json_files = list(Path.cwd().rglob("backtests/*/*.json"))
result_json_files = [
f for f in backtest_json_files
if not f.name.endswith("-order-events.json")
and not f.name.endswith("alpha-results.json")
]
if len(result_json_files) == 0:
raise MoreInfoError(
"Could not find a recent backtest result file, please use the --backtest-results option",
"https://www.lean.io/docs/lean-cli/backtesting/report#02-Generate-a-Report"
)
backtest_results = sorted(result_json_files,
key=lambda f: f.stat().st_mtime,
reverse=True)[0]
logger = container.logger()
if live_results is None:
logger.info(f"Generating a report from '{backtest_results}'")
else:
logger.info(
f"Generating a report from '{backtest_results}' and '{live_results}'"
)
project_directory = _find_project_directory(backtest_results)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder":
"/Lean/Data",
"strategy-name":
strategy_name or "",
"strategy-version":
strategy_version or "",
"strategy-description":
strategy_description or "",
"live-data-source-file":
"live-data-source-file.json" if live_results is not None else "",
"backtest-data-source-file":
"backtest-data-source-file.json",
"report-destination":
"/tmp/report.html",
"environment":
"report",
"log-handler":
"QuantConnect.Logging.CompositeLogHandler",
"messaging-handler":
"QuantConnect.Messaging.Messaging",
"job-queue-handler":
"QuantConnect.Queues.JobQueue",
"api-handler":
"QuantConnect.Api.Api",
"map-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider":
"QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler":
"QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider":
"DataChannelProvider",
"environments": {
"report": {
"live-mode":
False,
"setup-handler":
"QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler":
"QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler":
"QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler":
"QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider":
"QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler":
"QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
config_path = container.temp_manager().create_temporary_directory(
) / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
backtest_id = container.output_config_manager().get_backtest_id(
backtest_results.parent)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
report_destination.parent.mkdir(parents=True, exist_ok=True)
run_options: Dict[str, Any] = {
"detach":
detach,
"name":
f"lean_cli_report_{backtest_id}",
"working_dir":
"/Lean/Report/bin/Debug",
"commands": [
"dotnet QuantConnect.Report.dll",
f'cp /tmp/report.html "/Output/{report_destination.name}"'
],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(
target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_results),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
},
str(report_destination.parent): {
"bind": "/Output",
"mode": "rw"
}
}
}
if live_results is not None:
run_options["mounts"].append(
Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_results),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image_override = image
if engine_image_override is None and project_directory is not None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
engine_image_override = project_config.get("engine-image", None)
engine_image = cli_config_manager.get_engine_image(engine_image_override)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information"
)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started the report creator in the '{run_options['name']}' container"
)
logger.info(f"The report will be generated to '{report_destination}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
return
logger.info(f"Successfully generated report to '{report_destination}'")
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"commands": [],
"environment": {},
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_output": lambda chunk: _check_docker_output(chunk, port)
}
lean_runner = container.lean_runner()
if project_config.get("algorithm-language", "Python") == "Python":
lean_runner.set_up_python_options(project, "/Lean/Launcher/bin/Debug/Notebooks", run_options)
else:
lean_runner.set_up_csharp_options(project, run_options)
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and "port is already allocated" in msg:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
