def live(project: Path, environment: str, output: Optional[Path],
image: Optional[str], update: bool) -> None:
"""Start live trading a project locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
ENVIRONMENT must be the name of an environment in the Lean configuration file with live-mode set to true.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(Path(project))
if output is None:
output = algorithm_file.parent / "live" / datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S")
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
environment, algorithm_file, None)
if "environments" not in lean_config or environment not in lean_config[
"environments"]:
lean_config_path = lean_config_manager.get_lean_config_path()
raise MoreInfoError(
f"{lean_config_path} does not contain an environment named '{environment}'",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
if not lean_config["environments"][environment]["live-mode"]:
raise MoreInfoError(
f"The '{environment}' is not a live trading environment (live-mode is set to false)",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading"
)
_raise_for_missing_properties(lean_config, environment,
lean_config_manager.get_lean_config_path())
_start_iqconnect_if_necessary(lean_config, environment)
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
lean_runner = container.lean_runner()
lean_runner.run_lean(environment, algorithm_file, output, engine_image,
None)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def create_project(name: str, language: str) -> None:
"""Create a new project containing starter code.
If NAME is a path containing subdirectories those will be created automatically.
The default language can be set using `lean config set default-language python/csharp`.
"""
cli_config_manager = container.cli_config_manager()
language = language if language is not None else cli_config_manager.default_language.get_value(
)
if language is None:
raise MoreInfoError(
"Please specify a language with --language or set the default language using `lean config set default-language python/csharp`",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/project-management"
)
full_path = Path.cwd() / name
path_validator = container.path_validator()
if not path_validator.is_path_valid(full_path):
raise MoreInfoError(
f"'{name}' is not a valid path",
"https://www.quantconnect.com/docs/v2/lean-cli/user-guides/troubleshooting#02-Common-errors"
)
if full_path.exists():
raise RuntimeError(
f"A project named '{name}' already exists, please choose a different name"
)
else:
project_manager = container.project_manager()
project_manager.create_new_project(
full_path,
QCLanguage.Python if language == "python" else QCLanguage.CSharp)
# Convert the project name into a valid class name by removing all non-alphanumeric characters
class_name = re.sub(f"[^a-zA-Z0-9]", "",
"".join(map(_capitalize, name.split(" "))))
if language == "python":
with (full_path / "main.py").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_MAIN.replace("$NAME$", class_name))
else:
with (full_path / "Main.cs").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_CSHARP_MAIN.replace("$NAME$", class_name))
with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_NOTEBOOK if language ==
"python" else DEFAULT_CSHARP_NOTEBOOK)
with (full_path / "config.json").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_CONFIG if language ==
"python" else DEFAULT_CSHARP_CONFIG)
logger = container.logger()
logger.info(
f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'"
)
def create_project(name: str, language: str) -> None:
"""Create a new project containing starter code.
If NAME is a path containing subdirectories those will be created automatically.
The default language can be set using `lean config set default-language python/csharp`.
"""
cli_config_manager = container.cli_config_manager()
language = language if language is not None else cli_config_manager.default_language.get_value()
if language is None:
raise MoreInfoError(
"Please specify a language with --language or set the default language using `lean config set default-language python/csharp`",
"https://www.lean.io/docs/lean-cli/projects/project-management")
full_path = Path.cwd() / name
if not container.path_manager().is_path_valid(full_path):
raise MoreInfoError(f"'{name}' is not a valid path",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors")
is_library_project = False
try:
library_dir = container.lean_config_manager().get_cli_root_directory() / "Library"
is_library_project = library_dir in full_path.parents
except:
# get_cli_root_directory() raises an error if there is no such directory
pass
if is_library_project and language == "python" and not full_path.name.isidentifier():
raise RuntimeError(
f"'{full_path.name}' is not a valid Python identifier, which is required for Python library projects to be importable")
if full_path.exists():
raise RuntimeError(f"A project named '{name}' already exists, please choose a different name")
else:
project_manager = container.project_manager()
project_manager.create_new_project(full_path, QCLanguage.Python if language == "python" else QCLanguage.CSharp)
# Convert the project name into a valid class name by removing all non-alphanumeric characters
class_name = re.sub(f"[^a-zA-Z0-9]", "", "".join(map(_capitalize, full_path.name.split(" "))))
if language == "python":
main_name = "main.py"
main_content = DEFAULT_PYTHON_MAIN if not is_library_project else LIBRARY_PYTHON_MAIN
else:
main_name = "Main.cs"
main_content = DEFAULT_CSHARP_MAIN if not is_library_project else LIBRARY_CSHARP_MAIN
with (full_path / main_name).open("w+", encoding="utf-8") as file:
file.write(main_content.replace("$CLASS_NAME$", class_name).replace("$PROJECT_NAME$", full_path.name))
with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_NOTEBOOK if language == "python" else DEFAULT_CSHARP_NOTEBOOK)
logger = container.logger()
logger.info(f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'")
def __init__(self, username: str, account_id: str, account_password: str,
use_ib_feed: bool) -> None:
self._username = username
self._account_id = account_id
self._account_password = account_password
self._account_type = None
self._environment = None
self._use_ib_feed = use_ib_feed
demo_slice = account_id.lower()[:2]
live_slice = account_id.lower()[0]
if live_slice == "d":
if demo_slice == "df" or demo_slice == "du":
self._account_type = "individual"
self._environment = "paper"
elif demo_slice == "di":
self._account_type = "advisor"
self._environment = "paper"
else:
if live_slice == "f" or live_slice == "i":
self._account_type = "advisor"
self._environment = "live"
elif live_slice == "u":
self._account_type = "individual"
self._environment = "live"
if self._environment is None:
raise MoreInfoError(
f"Account id '{account_id}' does not look like a valid account name",
"https://www.lean.io/docs/lean-cli/live-trading/interactive-brokers#03-Deploy-Cloud-Algorithms"
)
def remove(project: Path, name: str, no_local: bool) -> None:
"""Remove a custom library from a project.
PROJECT must be the path to the project directory.
NAME must be the name of the NuGet package (for C# projects) or of the PyPI package (for Python projects) to remove.
Custom C# libraries are removed from the project's .csproj file,
which is then restored if dotnet is on your PATH and the --no-local flag has not been given.
Custom Python libraries are removed from the project's requirements.txt file.
\b
C# example usage:
$ lean library remove "My CSharp Project" Microsoft.ML
\b
Python example usage:
$ lean library remove "My Python Project" tensorflow
"""
project_config = container.project_config_manager().get_project_config(
project)
project_language = project_config.get("algorithm-language", None)
if project_language is None:
raise MoreInfoError(
f"{project} is not a Lean CLI project",
"https://www.lean.io/docs/lean-cli/tutorials/project-management#02-Creating-new-projects"
)
if project_language == "CSharp":
_remove_csharp(project, name, no_local)
else:
_remove_python(project, name)
def get_lean_config_path(self) -> Path:
"""Returns the path to the closest Lean config file.
This recurses upwards in the directory tree looking for a Lean config file.
This search can be overridden using set_default_lean_config_path().
Raises an error if no Lean config file can be found.
:return: the path to the closest Lean config file
"""
if self._default_path is not None:
return self._default_path
if self._lean_config_path is not None:
return self._lean_config_path
# Recurse upwards in the directory tree until we find a Lean config file
current_dir = Path.cwd()
while True:
target_file = current_dir / DEFAULT_LEAN_CONFIG_FILE_NAME
if target_file.exists():
self._lean_config_path = target_file
self.store_known_lean_config_path(self._lean_config_path)
return self._lean_config_path
# If the parent directory is the same as the current directory we can't go up any more
if current_dir.parent == current_dir:
raise MoreInfoError(
f"'{DEFAULT_LEAN_CONFIG_FILE_NAME}' not found",
"https://www.lean.io/docs/lean-cli/initialization/configuration#03-Lean-Configuration"
)
current_dir = current_dir.parent
def _configure_credentials(cls, lean_config: Dict[str, Any],
logger: Logger) -> None:
logger.info("""
To use IB with LEAN you must disable two-factor authentication or only use IBKR Mobile.
This is done from your IB Account Manage Account -> Settings -> User Settings -> Security -> Secure Login System.
In the Secure Login System, deselect all options or only select "IB Key Security via IBKR Mobile".
Interactive Brokers Lite accounts do not support API trading.
""".strip())
username = click.prompt("Username",
cls._get_default(lean_config, "ib-user-name"))
account_id = click.prompt("Account id",
cls._get_default(lean_config, "ib-account"))
account_password = logger.prompt_password(
"Account password", cls._get_default(lean_config, "ib-password"))
agent_description = None
trading_mode = None
demo_slice = account_id.lower()[:2]
live_slice = account_id.lower()[0]
if live_slice == "d":
if demo_slice == "df" or demo_slice == "du":
agent_description = "Individual"
trading_mode = "paper"
elif demo_slice == "di":
# TODO: Remove this once we know what ib-agent-description should be for Advisor accounts
raise RuntimeError(
"Please use the --environment option for Advisor accounts")
agent_description = "Advisor"
trading_mode = "paper"
else:
if live_slice == "f" or live_slice == "i":
# TODO: Remove this once we know what ib-agent-description should be for Advisor accounts
raise RuntimeError(
"Please use the --environment option for Advisor accounts")
agent_description = "Advisor"
trading_mode = "live"
elif live_slice == "u":
agent_description = "Individual"
trading_mode = "live"
if trading_mode is None:
raise MoreInfoError(
f"Account id '{account_id}' does not look like a valid account id",
"https://www.lean.io/docs/lean-cli/tutorials/live-trading/local-live-trading#03-Interactive-Brokers"
)
lean_config["ib-user-name"] = username
lean_config["ib-account"] = account_id
lean_config["ib-password"] = account_password
lean_config["ib-agent-description"] = agent_description
lean_config["ib-trading-mode"] = trading_mode
cls._save_properties(lean_config, [
"ib-user-name", "ib-account", "ib-password",
"ib-agent-description", "ib-trading-mode"
])
def invoke(self, ctx):
if self._requires_lean_config:
try:
# This method will throw if the directory cannot be found
container.lean_config_manager().get_cli_root_directory()
except Exception:
# Abort with a display-friendly error message if the command requires a Lean config
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/user-guides/troubleshooting#02-Common-errors")
if self._requires_docker and "pytest" not in sys.modules:
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if platform.system() == "Linux":
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if platform.system() == "Linux" and os.getuid() != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password")
args = ["sudo", "--preserve-env=HOME", sys.executable, *sys.argv]
os.execlp(args[0], *args)
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def get_option_by_key(self, key: str) -> Option:
"""Returns the option matching the given key.
If no option with the given key exists, an error is raised.
:param key: the key to look for
:return: the option having a key equal to the given key
"""
option = next((x for x in self.all_options if x.key == key), None)
if option is None:
raise MoreInfoError(f"There doesn't exist an option with key '{key}'",
"https://www.lean.io/docs/lean-cli/api-reference/lean-config-set#02-Description")
return option
def _raise_for_missing_properties(lean_config: Dict[str, Any],
environment_name: str,
lean_config_path: Path) -> None:
"""Raises an error if any required properties are missing.
:param lean_config: the LEAN configuration that should be used
:param environment_name: the name of the environment
:param lean_config_path: the path to the LEAN configuration file
"""
environment = lean_config["environments"][environment_name]
for key in ["live-mode-brokerage", "data-queue-handler"]:
if key not in environment:
raise MoreInfoError(
f"The '{environment_name}' environment does not specify a {key}",
"https://www.lean.io/docs/lean-cli/tutorials/live-trading/local-live-trading"
)
brokerage = environment["live-mode-brokerage"]
data_queue_handler = environment["data-queue-handler"]
brokerage_properties = _required_brokerage_properties.get(brokerage, [])
data_queue_handler_properties = _required_data_queue_handler_properties.get(
data_queue_handler, [])
required_properties = brokerage_properties + data_queue_handler_properties
missing_properties = [
p for p in required_properties
if p not in lean_config or lean_config[p] == ""
]
missing_properties = set(missing_properties)
if len(missing_properties) == 0:
return
properties_str = "properties" if len(
missing_properties) > 1 else "property"
these_str = "these" if len(missing_properties) > 1 else "this"
missing_properties = "\n".join(f"- {p}" for p in missing_properties)
raise RuntimeError(f"""
Please configure the following missing {properties_str} in {lean_config_path}:
{missing_properties}
Go to the following url for documentation on {these_str} {properties_str}:
https://www.lean.io/docs/lean-cli/tutorials/live-trading/local-live-trading
""".strip())
def add(project: Path, name: str, version: Optional[str],
no_local: bool) -> None:
"""Add a custom library to a project.
PROJECT must be the path to the project.
NAME must be the name of a NuGet package (for C# projects) or of a PyPI package (for Python projects).
If --version is not given, the package is pinned to the latest compatible version.
For C# projects, this is the latest available version.
For Python projects, this is the latest version compatible with Python 3.6 (which is what the Docker images use).
Custom C# libraries are added to your project's .csproj file,
which is then restored if dotnet is on your PATH and the --no-local flag has not been given.
Custom Python libraries are added to your project's requirements.txt file and are installed in your local Python
environment so you get local autocomplete for the library. The last step can be skipped with the --no-local flag.
\b
C# example usage:
$ lean library add "My CSharp Project" Microsoft.ML
$ lean library add "My CSharp Project" Microsoft.ML --version 1.5.5
\b
Python example usage:
$ lean library add "My Python Project" tensorflow
$ lean library add "My Python Project" tensorflow --version 2.5.0
"""
project_config = container.project_config_manager().get_project_config(
project)
project_language = project_config.get("algorithm-language", None)
if project_language is None:
raise MoreInfoError(
f"{project} is not a Lean CLI project",
"https://www.lean.io/docs/lean-cli/projects/project-management#02-Create-Projects"
)
if project_language == "CSharp":
_add_csharp(project, name, version, no_local)
else:
_add_python(project, name, version, no_local)
def cfd(ticker: str, resolution: str, start: Optional[datetime],
end: Optional[datetime], overwrite: bool) -> None:
"""Download free CFD data from QuantConnect's Data Library.
\b
This command can only download data that you have previously added to your QuantConnect account.
See the following url for instructions on how to do so:
https://www.quantconnect.com/docs/v2/lean-cli/tutorials/local-data/downloading-from-quantconnect#02-QuantConnect-Data-Library
\b
See the following url for the data that can be downloaded with this command:
https://www.quantconnect.com/data/tree/cfd/oanda
\b
Example of downloading https://www.quantconnect.com/data/file/cfd/oanda/daily/au200aud.zip/au200aud.csv:
$ lean download cfd --ticker au200aud --resolution daily
"""
ticker = ticker.lower()
if resolution == "hour" or resolution == "daily":
start = None
end = None
path_template = f"cfd/oanda/{resolution}/{ticker}.zip"
else:
if start is None or end is None:
raise MoreInfoError(
f"Both --start and --end must be given for {resolution} resolution",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/local-data/downloading-from-quantconnect#03-Downloading-data-from-Data-Library"
)
path_template = f"cfd/oanda/{resolution}/{ticker}/$DAY$_quote.zip"
data_downloader = container.data_downloader()
data_downloader.download_data(security_type=QCSecurityType.CFD,
ticker=ticker,
market="oanda",
resolution=QCResolution.by_name(resolution),
start=start,
end=end,
path_template=path_template,
overwrite=overwrite)
def forex(ticker: str, market: str, resolution: str, start: Optional[datetime],
end: Optional[datetime], overwrite: bool) -> None:
"""Download free Forex data from QuantConnect's Data Library.
\b
This command can only download data that you have previously added to your QuantConnect account.
See the following url for instructions on how to do so:
https://www.quantconnect.com/docs/v2/lean-cli/tutorials/local-data/downloading-from-quantconnect#02-QuantConnect-Data-Library
\b
See the following url for the data that can be downloaded with this command:
https://www.quantconnect.com/data/tree/forex
\b
Example of downloading 2019 data of https://www.quantconnect.com/data/tree/forex/fxcm/minute/eurusd:
$ lean download forex --ticker eurusd --market fxcm --resolution minute --start 20190101 --end 20191231
"""
ticker = ticker.lower()
if resolution == "hour" or resolution == "daily":
start = None
end = None
path_template = f"forex/{market}/{resolution}/{ticker}.zip"
else:
if start is None or end is None:
raise MoreInfoError(
f"Both --start and --end must be given for {resolution} resolution",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/local-data/downloading-from-quantconnect#03-Downloading-data-from-Data-Library"
)
path_template = f"forex/{market}/{resolution}/{ticker}/$DAY$_quote.zip"
data_downloader = container.data_downloader()
data_downloader.download_data(security_type=QCSecurityType.Forex,
ticker=ticker,
market=market,
resolution=QCResolution.by_name(resolution),
start=start,
end=end,
path_template=path_template,
overwrite=overwrite)
def _get_docker_client(self) -> docker.DockerClient:
"""Creates a DockerClient instance.
Raises an error if Docker is not running.
:return: a DockerClient instance which responds to requests
"""
error = MoreInfoError("Please make sure Docker is installed and running",
"https://www.quantconnect.com/docs/v2/lean-cli/user-guides/troubleshooting#02-Common-errors")
try:
docker_client = docker.from_env()
except Exception:
raise error
try:
if not docker_client.ping():
raise error
except Exception:
raise error
return docker_client
def _get_settings(self, logger: Logger) -> Dict[str, str]:
username = click.prompt("Username")
account_id = click.prompt("Account id")
account_password = logger.prompt_password("Account password")
account_type = None
environment = None
demo_slice = account_id.lower()[:2]
live_slice = account_id.lower()[0]
if live_slice == "d":
if demo_slice == "df" or demo_slice == "du":
account_type = "individual"
environment = "paper"
elif demo_slice == "di":
account_type = "advisor"
environment = "paper"
else:
if live_slice == "f" or live_slice == "i":
account_type = "advisor"
environment = "live"
elif live_slice == "u":
account_type = "individual"
environment = "live"
if environment is None:
raise MoreInfoError(
f"Account id '{account_id}' does not look like a valid account name",
"https://www.lean.io/docs/lean-cli/tutorials/live-trading/cloud-live-trading#03-Interactive-Brokers"
)
return {
"user": username,
"account": account_id,
"password": account_password,
"accountType": account_type,
"environment": environment
}
def get(key: str) -> None:
"""Get the current value of a configurable option.
Sensitive options like credentials cannot be retrieved this way for security reasons.
Please open ~/.lean/credentials if you want to see your currently stored credentials.
Run `lean config list` to show all available options.
"""
cli_config_manager = container.cli_config_manager()
option = cli_config_manager.get_option_by_key(key)
if option.is_sensitive:
raise RuntimeError(
"Sensitive options like credentials cannot be retrieved using `lean config get` for security reasons")
value = option.get_value()
if value is None:
raise MoreInfoError(f"The option with key '{key}' doesn't have a value set",
"https://www.quantconnect.com/docs/v2/lean-cli/api-reference/lean-config-set")
logger = container.logger()
logger.info(value)
def login(user_id: Optional[str], api_token: Optional[str]) -> None:
"""Log in with a QuantConnect account.
If user id or API token is not provided an interactive prompt will show.
Credentials are stored in ~/.lean/credentials and are removed upon running `lean logout`.
"""
logger = container.logger()
credentials_storage = container.credentials_storage()
if user_id is None or api_token is None:
logger.info(
"Your user id and API token are needed to make authenticated requests to the QuantConnect API"
)
logger.info(
"You can request these credentials on https://www.quantconnect.com/account"
)
logger.info(f"Both will be saved in {credentials_storage.file}")
if user_id is None:
user_id = click.prompt("User id")
if api_token is None:
api_token = logger.prompt_password("API token")
api_client = container.api_client(user_id=user_id, api_token=api_token)
if not api_client.is_authenticated():
raise MoreInfoError(
"Credentials are invalid",
"https://www.lean.io/docs/lean-cli/tutorials/authentication#02-Logging-in"
)
cli_config_manager = container.cli_config_manager()
cli_config_manager.user_id.set_value(user_id)
cli_config_manager.api_token.set_value(api_token)
logger.info("Successfully logged in")
def __init__(self, username: str, account_id: str,
account_password: str) -> None:
self._username = username
self._account_id = account_id
self._account_password = account_password
self._agent_description = None
self._trading_mode = None
demo_slice = account_id.lower()[:2]
live_slice = account_id.lower()[0]
if live_slice == "d":
if demo_slice == "df" or demo_slice == "du":
self._agent_description = "Individual"
self._trading_mode = "paper"
elif demo_slice == "di":
# TODO: Remove this once we know what ib-agent-description should be for Advisor accounts
raise RuntimeError(
"Please use the --environment option for Advisor accounts")
self._agent_description = "Advisor"
self._trading_mode = "paper"
else:
if live_slice == "f" or live_slice == "i":
# TODO: Remove this once we know what ib-agent-description should be for Advisor accounts
raise RuntimeError(
"Please use the --environment option for Advisor accounts")
self._agent_description = "Advisor"
self._trading_mode = "live"
elif live_slice == "u":
self._agent_description = "Individual"
self._trading_mode = "live"
if self._trading_mode is None:
raise MoreInfoError(
f"Account id '{account_id}' does not look like a valid account id",
"https://www.lean.io/docs/lean-cli/live-trading/interactive-brokers#02-Deploy-Local-Algorithms"
)
def optimize(project: Path, output: Optional[Path], detach: bool,
optimizer_config: Optional[Path], strategy: Optional[str],
target: Optional[str], target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
release: bool, image: Optional[str], update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
By default an interactive wizard is shown letting you configure the optimizer.
If --optimizer-config or --strategy is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
If --strategy is given the optimizer configuration is read from the given options.
In this case --strategy, --target, --target-direction and --parameter become required.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
optimizer_config_manager = container.optimizer_config_manager()
config = None
if optimizer_config is not None and strategy is not None:
raise RuntimeError(
"--optimizer-config and --strategy are mutually exclusive")
if optimizer_config is not None:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
elif strategy is not None:
ensure_options(["strategy", "target", "target_direction", "parameter"])
optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
else:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/optimization/parameters")
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
if config is None:
# noinspection PyUnboundLocalVariable
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_optimization_id(output))
lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging"
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None, release, detach)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
project_manager.copy_code(algorithm_file.parent, output / "code")
success = container.docker_manager().run_image(engine_image, **run_options)
logger = container.logger()
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(f"The output will be stored in '{relative_output_dir}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
elif success:
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
def invoke(self, ctx: click.Context):
if self._requires_lean_config:
lean_config_manager = container.lean_config_manager()
try:
# This method will raise an error if the directory cannot be found
lean_config_manager.get_cli_root_directory()
except Exception:
# Use one of the cached Lean config locations to avoid having to abort the command
lean_config_paths = lean_config_manager.get_known_lean_config_paths(
)
if len(lean_config_paths) > 0:
lean_config_path = container.logger().prompt_list(
"Select the Lean configuration file to use", [
Option(id=p, label=str(p))
for p in lean_config_paths
])
lean_config_manager.set_default_lean_config_path(
lean_config_path)
else:
# Abort with a display-friendly error message if the command requires a Lean config and none found
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors"
)
if self._requires_docker and "pytest" not in sys.modules:
is_system_linux = container.platform_manager().is_system_linux()
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if is_system_linux:
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if is_system_linux and os.getuid(
) != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password"
)
args = [
"sudo", "--preserve-env=HOME", sys.executable, *sys.argv
]
os.execlp(args[0], *args)
if self._allow_unknown_options:
# Unknown options are passed to ctx.args and need to be parsed manually
# We parse them to ctx.params so they're available like normal options
# Because of this all commands with allow_unknown_options=True must have a **kwargs argument
arguments = list(
itertools.chain(*[arg.split("=") for arg in ctx.args]))
skip_next = False
for index in range(len(arguments) - 1):
if skip_next:
skip_next = False
continue
if arguments[index].startswith("--"):
option = arguments[index].replace("--", "")
value = arguments[index + 1]
ctx.params[option] = value
skip_next = True
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def report(backtest_results: Optional[Path], live_results: Optional[Path],
report_destination: Path, detach: bool,
strategy_name: Optional[str], strategy_version: Optional[str],
strategy_description: Optional[str], overwrite: bool,
image: Optional[str], update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
If --backtest-results is not given, a report is generated for the most recent local backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(
f"{report_destination} already exists, use --overwrite to overwrite it"
)
if backtest_results is None:
backtest_json_files = list(Path.cwd().rglob("backtests/*/*.json"))
result_json_files = [
f for f in backtest_json_files
if not f.name.endswith("-order-events.json")
and not f.name.endswith("alpha-results.json")
]
if len(result_json_files) == 0:
raise MoreInfoError(
"Could not find a recent backtest result file, please use the --backtest-results option",
"https://www.lean.io/docs/lean-cli/backtesting/report#02-Generate-a-Report"
)
backtest_results = sorted(result_json_files,
key=lambda f: f.stat().st_mtime,
reverse=True)[0]
logger = container.logger()
if live_results is None:
logger.info(f"Generating a report from '{backtest_results}'")
else:
logger.info(
f"Generating a report from '{backtest_results}' and '{live_results}'"
)
project_directory = _find_project_directory(backtest_results)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder":
"/Lean/Data",
"strategy-name":
strategy_name or "",
"strategy-version":
strategy_version or "",
"strategy-description":
strategy_description or "",
"live-data-source-file":
"live-data-source-file.json" if live_results is not None else "",
"backtest-data-source-file":
"backtest-data-source-file.json",
"report-destination":
"/tmp/report.html",
"environment":
"report",
"log-handler":
"QuantConnect.Logging.CompositeLogHandler",
"messaging-handler":
"QuantConnect.Messaging.Messaging",
"job-queue-handler":
"QuantConnect.Queues.JobQueue",
"api-handler":
"QuantConnect.Api.Api",
"map-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider":
"QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler":
"QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider":
"DataChannelProvider",
"environments": {
"report": {
"live-mode":
False,
"setup-handler":
"QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler":
"QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler":
"QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler":
"QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider":
"QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler":
"QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
config_path = container.temp_manager().create_temporary_directory(
) / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
backtest_id = container.output_config_manager().get_backtest_id(
backtest_results.parent)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
report_destination.parent.mkdir(parents=True, exist_ok=True)
run_options: Dict[str, Any] = {
"detach":
detach,
"name":
f"lean_cli_report_{backtest_id}",
"working_dir":
"/Lean/Report/bin/Debug",
"commands": [
"dotnet QuantConnect.Report.dll",
f'cp /tmp/report.html "/Output/{report_destination.name}"'
],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(
target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_results),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
},
str(report_destination.parent): {
"bind": "/Output",
"mode": "rw"
}
}
}
if live_results is not None:
run_options["mounts"].append(
Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_results),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image_override = image
if engine_image_override is None and project_directory is not None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
engine_image_override = project_config.get("engine-image", None)
engine_image = cli_config_manager.get_engine_image(engine_image_override)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information"
)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started the report creator in the '{run_options['name']}' container"
)
logger.info(f"The report will be generated to '{report_destination}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
return
logger.info(f"Successfully generated report to '{report_destination}'")
def init() -> None:
"""Scaffold a Lean configuration file and data directory."""
current_dir = Path.cwd()
data_dir = current_dir / DEFAULT_DATA_DIRECTORY_NAME
lean_config_path = current_dir / DEFAULT_LEAN_CONFIG_FILE_NAME
# Abort if one of the files we are going to create already exists to prevent us from overriding existing files
for path in [data_dir, lean_config_path]:
if path.exists():
relative_path = path.relative_to(current_dir)
raise MoreInfoError(f"{relative_path} already exists, please run this command in an empty directory",
"https://www.lean.io/docs/lean-cli/initialization/directory-structure#02-lean-init")
logger = container.logger()
# Warn the user if the current directory is not empty
if next(current_dir.iterdir(), None) is not None:
logger.info("This command will create a Lean configuration file and data directory in the current directory")
click.confirm("The current directory is not empty, continue?", default=False, abort=True)
# Download the Lean repository
tmp_directory = container.temp_manager().create_temporary_directory()
_download_repository(tmp_directory / "master.zip")
# Extract the downloaded repository
with zipfile.ZipFile(tmp_directory / "master.zip") as zip_file:
zip_file.extractall(tmp_directory / "master")
# Copy the data directory
shutil.copytree(tmp_directory / "master" / "Lean-master" / "Data", data_dir)
# Create the config file
lean_config_manager = container.lean_config_manager()
config = (tmp_directory / "master" / "Lean-master" / "Launcher" / "config.json").read_text(encoding="utf-8")
config = lean_config_manager.clean_lean_config(config)
lean_config_manager.store_known_lean_config_path(lean_config_path)
# Update the data-folder configuration
config = config.replace('"data-folder": "../../../Data/"', f'"data-folder": "{DEFAULT_DATA_DIRECTORY_NAME}"')
with lean_config_path.open("w+", encoding="utf-8") as file:
file.write(config)
# Prompt for some general configuration if not set yet
cli_config_manager = container.cli_config_manager()
if cli_config_manager.default_language.get_value() is None:
default_language = click.prompt("What should the default language for new projects be?",
type=click.Choice(cli_config_manager.default_language.allowed_values))
cli_config_manager.default_language.set_value(default_language)
logger.info(f"""
The following objects have been created:
- {DEFAULT_LEAN_CONFIG_FILE_NAME} contains the configuration used when running the LEAN engine locally
- {DEFAULT_DATA_DIRECTORY_NAME}/ contains the data that is used when running the LEAN engine locally
The following documentation pages may be useful:
- Setting up local autocomplete: https://www.lean.io/docs/lean-cli/projects/autocomplete
- Synchronizing projects with the cloud: https://www.lean.io/docs/lean-cli/projects/cloud-synchronization
Here are some commands to get you going:
- Run `lean create-project "My Project"` to create a new project with starter code
- Run `lean cloud pull` to download all your QuantConnect projects to your local drive
- Run `lean backtest "My Project"` to backtest a project locally with the data in {DEFAULT_DATA_DIRECTORY_NAME}/
""".strip())
# Prompt to create a desktop shortcut for the local GUI if the user is in an organization with a subscription
api_client = container.api_client()
if api_client.is_authenticated():
for simple_organization in api_client.organizations.get_all():
organization = api_client.organizations.get(simple_organization.id)
modules_product = next((p for p in organization.products if p.name == "Modules"), None)
if modules_product is None:
continue
if any(i for i in modules_product.items if i.productId in GUI_PRODUCT_SUBSCRIPTION_IDS):
container.shortcut_manager().prompt_if_necessary(simple_organization.id)
break
def optimize(project: Path, output: Optional[Path],
optimizer_config: Optional[Path], image: Optional[str],
update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
When --optimizer-config is not set, an interactive prompt will be shown to configure the optimizer.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
if optimizer_config is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/tutorials/optimization/project-parameters"
)
optimizer_config_manager = container.optimizer_config_manager()
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
else:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None, None)
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if success:
logger = container.logger()
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)