def main() -> None:
"""This function is the entrypoint when running a Lean command in a terminal."""
try:
lean.main(standalone_mode=False)
temp_manager = container.temp_manager()
if temp_manager.delete_temporary_directories_when_done:
temp_manager.delete_temporary_directories()
except Exception as exception:
logger = container.logger()
logger.debug(traceback.format_exc().strip())
if isinstance(exception, ValidationError) and hasattr(exception, "input_value"):
logger.debug("Value that failed validation:")
logger.debug(exception.input_value)
logger.error(f"Error: {exception}")
elif isinstance(exception, MoreInfoError):
logger.error(f"Error: {exception}")
logger.error(f"Visit {exception.link} for more information")
elif isinstance(exception, click.UsageError):
io = StringIO()
exception.show(file=io)
exception_str = io.getvalue().strip()
exception_str = exception_str.replace("Try 'lean", "\nTry 'lean")
exception_str = exception_str.replace("for help.",
"for help or go to the following url for a list of common errors:\nhttps://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors")
container.update_manager().warn_if_cli_outdated(force=True)
logger.info(exception_str)
elif isinstance(exception, click.Abort):
logger.info("Aborted!")
elif isinstance(exception, requests.exceptions.ConnectionError):
logger.error(f"Error: {exception}")
logger.error("It looks like you don't have an internet connection, please check your system settings")
else:
logger.error(f"Error: {exception}")
temp_manager = container.temp_manager()
if temp_manager.delete_temporary_directories_when_done:
temp_manager.delete_temporary_directories()
sys.exit(1)
def init() -> None:
"""Scaffold a Lean configuration file and data directory."""
current_dir = Path.cwd()
data_dir = current_dir / DEFAULT_DATA_DIRECTORY_NAME
lean_config_path = current_dir / DEFAULT_LEAN_CONFIG_FILE_NAME
# Abort if one of the files we are going to create already exists to prevent us from overriding existing files
for path in [data_dir, lean_config_path]:
if path.exists():
relative_path = path.relative_to(current_dir)
raise MoreInfoError(f"{relative_path} already exists, please run this command in an empty directory",
"https://www.lean.io/docs/lean-cli/initialization/directory-structure#02-lean-init")
logger = container.logger()
# Warn the user if the current directory is not empty
if next(current_dir.iterdir(), None) is not None:
logger.info("This command will create a Lean configuration file and data directory in the current directory")
click.confirm("The current directory is not empty, continue?", default=False, abort=True)
# Download the Lean repository
tmp_directory = container.temp_manager().create_temporary_directory()
_download_repository(tmp_directory / "master.zip")
# Extract the downloaded repository
with zipfile.ZipFile(tmp_directory / "master.zip") as zip_file:
zip_file.extractall(tmp_directory / "master")
# Copy the data directory
shutil.copytree(tmp_directory / "master" / "Lean-master" / "Data", data_dir)
# Create the config file
lean_config_manager = container.lean_config_manager()
config = (tmp_directory / "master" / "Lean-master" / "Launcher" / "config.json").read_text(encoding="utf-8")
config = lean_config_manager.clean_lean_config(config)
lean_config_manager.store_known_lean_config_path(lean_config_path)
# Update the data-folder configuration
config = config.replace('"data-folder": "../../../Data/"', f'"data-folder": "{DEFAULT_DATA_DIRECTORY_NAME}"')
with lean_config_path.open("w+", encoding="utf-8") as file:
file.write(config)
# Prompt for some general configuration if not set yet
cli_config_manager = container.cli_config_manager()
if cli_config_manager.default_language.get_value() is None:
default_language = click.prompt("What should the default language for new projects be?",
type=click.Choice(cli_config_manager.default_language.allowed_values))
cli_config_manager.default_language.set_value(default_language)
logger.info(f"""
The following objects have been created:
- {DEFAULT_LEAN_CONFIG_FILE_NAME} contains the configuration used when running the LEAN engine locally
- {DEFAULT_DATA_DIRECTORY_NAME}/ contains the data that is used when running the LEAN engine locally
The following documentation pages may be useful:
- Setting up local autocomplete: https://www.lean.io/docs/lean-cli/projects/autocomplete
- Synchronizing projects with the cloud: https://www.lean.io/docs/lean-cli/projects/cloud-synchronization
Here are some commands to get you going:
- Run `lean create-project "My Project"` to create a new project with starter code
- Run `lean cloud pull` to download all your QuantConnect projects to your local drive
- Run `lean backtest "My Project"` to backtest a project locally with the data in {DEFAULT_DATA_DIRECTORY_NAME}/
""".strip())
# Prompt to create a desktop shortcut for the local GUI if the user is in an organization with a subscription
api_client = container.api_client()
if api_client.is_authenticated():
for simple_organization in api_client.organizations.get_all():
organization = api_client.organizations.get(simple_organization.id)
modules_product = next((p for p in organization.products if p.name == "Modules"), None)
if modules_product is None:
continue
if any(i for i in modules_product.items if i.productId in GUI_PRODUCT_SUBSCRIPTION_IDS):
container.shortcut_manager().prompt_if_necessary(simple_organization.id)
break
def research(project: Path, port: int, data_provider: Optional[str],
download_data: bool, data_purchase_limit: Optional[int],
detach: bool, no_open: bool, image: Optional[str],
update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
lean_config["composer-dll-directory"] = "/Lean/Launcher/bin/Debug"
if download_data:
data_provider = QuantConnectDataProvider.get_name()
if data_provider is not None:
data_provider = next(dp for dp in all_data_providers
if dp.get_name() == data_provider)
data_provider.build(lean_config, container.logger()).configure(
lean_config, "backtesting")
lean_config_manager.configure_data_purchase_limit(lean_config,
data_purchase_limit)
lean_runner = container.lean_runner()
temp_manager = container.temp_manager()
run_options = lean_runner.get_basic_docker_config(
lean_config, algorithm_file, temp_manager.create_temporary_directory(),
None, False, detach)
# Mount the config in the notebooks directory as well
local_config_path = next(m["Source"] for m in run_options["mounts"]
if m["Target"].endswith("config.json"))
run_options["mounts"].append(
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(local_config_path),
type="bind",
read_only=True))
# Jupyter Lab runs on port 8888, we expose it to the local port specified by the user
run_options["ports"]["8888"] = str(port)
# Open the browser as soon as Jupyter Lab has started
if detach or not no_open:
run_options["on_output"] = lambda chunk: _check_docker_output(
chunk, port)
# Give container an identifiable name when running it from the GUI
if container.module_manager().is_module_installed(GUI_PRODUCT_INSTALL_ID):
project_id = container.project_config_manager().get_local_id(
algorithm_file.parent)
run_options["name"] = f"lean_cli_gui_research_{project_id}"
# Make Ctrl+C stop Jupyter Lab immediately
run_options["stop_signal"] = "SIGKILL"
# Mount the project to the notebooks directory
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Allow notebooks to be embedded in iframes
run_options["commands"].append("mkdir -p ~/.jupyter")
run_options["commands"].append(
'echo "c.NotebookApp.disable_check_xsrf = True\nc.NotebookApp.tornado_settings = {\'headers\': {\'Content-Security-Policy\': \'frame-ancestors self *\'}}" > ~/.jupyter/jupyter_notebook_config.py'
)
# Hide headers in notebooks
run_options["commands"].append(
"mkdir -p ~/.ipython/profile_default/static/custom")
run_options["commands"].append(
'echo "#header-container { display: none !important; }" > ~/.ipython/profile_default/static/custom/custom.css'
)
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
research_image = cli_config_manager.get_research_image(
image or project_config.get("research-image", None))
container.update_manager().pull_docker_image_if_necessary(
research_image, update)
try:
container.docker_manager().run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
raise RuntimeError(
f"Port {port} is already in use, please specify a different port using --port <number>"
)
raise error
if detach:
temp_manager.delete_temporary_directories_when_done = False
logger = container.logger()
relative_project_dir = algorithm_file.parent.relative_to(
lean_config_manager.get_cli_root_directory())
logger.info(
f"Successfully started Jupyter Lab environment for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(
"You can use Docker's own commands to manage the detached container"
)
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options = {
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
},
str(project): {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_run": lambda: webbrowser.open(f"http://localhost:{port}/")
}
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
if "port is already allocated" in error.explanation:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
def report(backtest_data_source_file: Path,
live_data_source_file: Optional[Path],
report_destination: Path,
strategy_name: Optional[str],
strategy_version: Optional[str],
strategy_description: Optional[str],
overwrite: bool,
image: Optional[str],
update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it")
project_directory = _find_project_directory(backtest_data_source_file)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder": "/Lean/Data",
"strategy-name": strategy_name or "",
"strategy-version": strategy_version or "",
"strategy-description": strategy_description or "",
"live-data-source-file": "live-data-source-file.json" if live_data_source_file is not None else "",
"backtest-data-source-file": "backtest-data-source-file.json",
"report-destination": "/Results/report.html",
"environment": "report",
"log-handler": "QuantConnect.Logging.CompositeLogHandler",
"messaging-handler": "QuantConnect.Messaging.Messaging",
"job-queue-handler": "QuantConnect.Queues.JobQueue",
"api-handler": "QuantConnect.Api.Api",
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler": "QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider": "DataChannelProvider",
"environments": {
"report": {
"live-mode": False,
"setup-handler": "QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler": "QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler": "QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider": "QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
output_dir = container.temp_manager().create_temporary_directory()
config_path = output_dir / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"working_dir": "/Lean/Report/bin/Debug",
"entrypoint": ["dotnet", "QuantConnect.Report.dll"],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_data_source_file),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "ro"
},
str(output_dir): {
"bind": "/Results",
"mode": "rw"
}
}
}
if live_data_source_file is not None:
run_options["mounts"].append(Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_data_source_file),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information")
report_destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output_dir / "report.html", report_destination)
logger = container.logger()
logger.info(f"Successfully generated report to '{report_destination}'")
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def report(backtest_results: Optional[Path], live_results: Optional[Path],
report_destination: Path, detach: bool,
strategy_name: Optional[str], strategy_version: Optional[str],
strategy_description: Optional[str], overwrite: bool,
image: Optional[str], update: bool) -> None:
"""Generate a report of a backtest.
This runs the LEAN Report Creator in Docker to generate a polished, professional-grade report of a backtest.
If --backtest-results is not given, a report is generated for the most recent local backtest.
The name, description, and version are optional and will be blank if not given.
If the given backtest data source file is stored in a project directory (or one of its subdirectories, like the
default <project>/backtests/<timestamp>), the default name is the name of the project directory and the default
description is the description stored in the project's config.json file.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
if report_destination.exists() and not overwrite:
raise RuntimeError(
f"{report_destination} already exists, use --overwrite to overwrite it"
)
if backtest_results is None:
backtest_json_files = list(Path.cwd().rglob("backtests/*/*.json"))
result_json_files = [
f for f in backtest_json_files
if not f.name.endswith("-order-events.json")
and not f.name.endswith("alpha-results.json")
]
if len(result_json_files) == 0:
raise MoreInfoError(
"Could not find a recent backtest result file, please use the --backtest-results option",
"https://www.lean.io/docs/lean-cli/backtesting/report#02-Generate-a-Report"
)
backtest_results = sorted(result_json_files,
key=lambda f: f.stat().st_mtime,
reverse=True)[0]
logger = container.logger()
if live_results is None:
logger.info(f"Generating a report from '{backtest_results}'")
else:
logger.info(
f"Generating a report from '{backtest_results}' and '{live_results}'"
)
project_directory = _find_project_directory(backtest_results)
if project_directory is not None:
if strategy_name is None:
strategy_name = project_directory.name
if strategy_description is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
strategy_description = project_config.get("description", "")
# The configuration given to the report creator
# See https://github.com/QuantConnect/Lean/blob/master/Report/config.example.json
report_config = {
"data-folder":
"/Lean/Data",
"strategy-name":
strategy_name or "",
"strategy-version":
strategy_version or "",
"strategy-description":
strategy_description or "",
"live-data-source-file":
"live-data-source-file.json" if live_results is not None else "",
"backtest-data-source-file":
"backtest-data-source-file.json",
"report-destination":
"/tmp/report.html",
"environment":
"report",
"log-handler":
"QuantConnect.Logging.CompositeLogHandler",
"messaging-handler":
"QuantConnect.Messaging.Messaging",
"job-queue-handler":
"QuantConnect.Queues.JobQueue",
"api-handler":
"QuantConnect.Api.Api",
"map-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
"factor-file-provider":
"QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
"data-provider":
"QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
"alpha-handler":
"QuantConnect.Lean.Engine.Alphas.DefaultAlphaHandler",
"data-channel-provider":
"DataChannelProvider",
"environments": {
"report": {
"live-mode":
False,
"setup-handler":
"QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler",
"result-handler":
"QuantConnect.Lean.Engine.Results.BacktestingResultHandler",
"data-feed-handler":
"QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed",
"real-time-handler":
"QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler",
"history-provider":
"QuantConnect.Lean.Engine.HistoricalData.SubscriptionDataReaderHistoryProvider",
"transaction-handler":
"QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"
}
}
}
config_path = container.temp_manager().create_temporary_directory(
) / "config.json"
with config_path.open("w+", encoding="utf-8") as file:
json.dump(report_config, file)
backtest_id = container.output_config_manager().get_backtest_id(
backtest_results.parent)
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
report_destination.parent.mkdir(parents=True, exist_ok=True)
run_options: Dict[str, Any] = {
"detach":
detach,
"name":
f"lean_cli_report_{backtest_id}",
"working_dir":
"/Lean/Report/bin/Debug",
"commands": [
"dotnet QuantConnect.Report.dll",
f'cp /tmp/report.html "/Output/{report_destination.name}"'
],
"mounts": [
Mount(target="/Lean/Report/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True),
Mount(
target="/Lean/Report/bin/Debug/backtest-data-source-file.json",
source=str(backtest_results),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Data",
"mode": "rw"
},
str(report_destination.parent): {
"bind": "/Output",
"mode": "rw"
}
}
}
if live_results is not None:
run_options["mounts"].append(
Mount(target="/Lean/Report/bin/Debug/live-data-source-file.json",
source=str(live_results),
type="bind",
read_only=True))
cli_config_manager = container.cli_config_manager()
engine_image_override = image
if engine_image_override is None and project_directory is not None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
project_directory)
engine_image_override = project_config.get("engine-image", None)
engine_image = cli_config_manager.get_engine_image(engine_image_override)
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
success = container.docker_manager().run_image(engine_image, **run_options)
if not success:
raise RuntimeError(
"Something went wrong while running the LEAN Report Creator, see the logs above for more information"
)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started the report creator in the '{run_options['name']}' container"
)
logger.info(f"The report will be generated to '{report_destination}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
return
logger.info(f"Successfully generated report to '{report_destination}'")
def research(project: Path, port: int, image: Optional[str], update: bool) -> None:
"""Run a Jupyter Lab environment locally using Docker.
By default the official LEAN research image is used.
You can override this using the --image option.
Alternatively you can set the default research image using `lean config set research-image <image>`.
"""
cli_config_manager = container.cli_config_manager()
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(project)
# Copy the config to a temporary config file before we add some research-specific configuration to it
config_path = container.temp_manager().create_temporary_directory() / "config.json"
project_config.file = config_path
project_config.set("composer-dll-directory", "/Lean/Launcher/bin/Debug")
project_config.set("messaging-handler", "QuantConnect.Messaging.Messaging")
project_config.set("job-queue-handler", "QuantConnect.Queues.JobQueue")
project_config.set("api-handler", "QuantConnect.Api.Api")
project_config.set("job-user-id", cli_config_manager.user_id.get_value("1"))
project_config.set("api-access-token", cli_config_manager.api_token.get_value("default"))
lean_config_manager = container.lean_config_manager()
data_dir = lean_config_manager.get_data_directory()
run_options: Dict[str, Any] = {
"commands": [],
"environment": {},
"mounts": [
Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json",
source=str(config_path),
type="bind",
read_only=True)
],
"volumes": {
str(data_dir): {
"bind": "/Lean/Launcher/Data",
"mode": "rw"
}
},
"ports": {
"8888": str(port)
},
"on_output": lambda chunk: _check_docker_output(chunk, port)
}
lean_runner = container.lean_runner()
if project_config.get("algorithm-language", "Python") == "Python":
lean_runner.set_up_python_options(project, "/Lean/Launcher/bin/Debug/Notebooks", run_options)
else:
lean_runner.set_up_csharp_options(project, run_options)
run_options["volumes"][str(project)] = {
"bind": "/Lean/Launcher/bin/Debug/Notebooks",
"mode": "rw"
}
# Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements
run_options["commands"].append(" && ".join([
'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx',
"mv NewQuantConnect.csx QuantConnect.csx"
]))
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")
cli_config_manager = container.cli_config_manager()
research_image = cli_config_manager.get_research_image(image)
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(research_image):
docker_manager.pull_image(research_image)
if str(research_image) == DEFAULT_RESEARCH_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(research_image)
try:
docker_manager.run_image(research_image, **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and "port is already allocated" in msg:
raise RuntimeError(f"Port {port} is already in use, please specify a different port using --port <number>")
raise error
def optimize(project: Path, output: Optional[Path], detach: bool,
optimizer_config: Optional[Path], strategy: Optional[str],
target: Optional[str], target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
release: bool, image: Optional[str], update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
By default an interactive wizard is shown letting you configure the optimizer.
If --optimizer-config or --strategy is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
If --strategy is given the optimizer configuration is read from the given options.
In this case --strategy, --target, --target-direction and --parameter become required.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
optimizer_config_manager = container.optimizer_config_manager()
config = None
if optimizer_config is not None and strategy is not None:
raise RuntimeError(
"--optimizer-config and --strategy are mutually exclusive")
if optimizer_config is not None:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
elif strategy is not None:
ensure_options(["strategy", "target", "target_direction", "parameter"])
optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
else:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/optimization/parameters")
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
if config is None:
# noinspection PyUnboundLocalVariable
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_optimization_id(output))
lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging"
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None, release, detach)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
project_manager.copy_code(algorithm_file.parent, output / "code")
success = container.docker_manager().run_image(engine_image, **run_options)
logger = container.logger()
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(f"The output will be stored in '{relative_output_dir}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
elif success:
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
def start(organization: Optional[str], port: int, no_open: bool,
shortcut: bool, gui: Optional[Path], shortcut_launch: bool) -> None:
"""Start the local GUI."""
logger = container.logger()
docker_manager = container.docker_manager()
temp_manager = container.temp_manager()
module_manager = container.module_manager()
api_client = container.api_client()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is not None:
if gui_container.status == "running":
if shortcut_launch:
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
webbrowser.open(url)
return
else:
_error(
"The local GUI is already running, run `lean gui restart` to restart it or `lean gui stop` to stop it",
shortcut_launch)
gui_container.remove()
if organization is not None:
organization_id = _get_organization_id(organization, shortcut_launch)
else:
organizations = api_client.organizations.get_all()
options = [
Option(id=organization.id, label=organization.name)
for organization in organizations
]
organization_id = logger.prompt_list(
"Select the organization with the local GUI module subscription",
options)
module_manager.install_module(GUI_PRODUCT_INSTALL_ID, organization_id)
shortcut_manager = container.shortcut_manager()
if shortcut:
shortcut_manager.create_shortcut(organization_id)
else:
shortcut_manager.prompt_if_necessary(organization_id)
# The dict containing all options passed to `docker run`
# See all available options at https://docker-py.readthedocs.io/en/stable/containers.html
run_options: Dict[str, Any] = {
"name": LOCAL_GUI_CONTAINER_NAME,
"detach": True,
"remove": False,
"commands": [],
"environment": {
"PYTHONUNBUFFERED": "1",
"QC_LOCAL_GUI": "true",
"QC_DOCKER_HOST_SYSTEM": platform.system(),
"QC_DOCKER_HOST_MACHINE": platform.machine(),
"QC_ORGANIZATION_ID": organization_id,
"QC_API": os.environ.get("QC_API", "")
},
"mounts": [],
"volumes": {},
"ports": {
"5612": str(port)
}
}
# Cache the site-packages so we don't re-install everything when the container is restarted
docker_manager.create_volume("lean_cli_gui_python")
run_options["volumes"]["lean_cli_gui_python"] = {
"bind": "/root/.local/lib/python3.9/site-packages",
"mode": "rw"
}
# Update PATH in the GUI container to add executables installed with pip
run_options["commands"].append('export PATH="$PATH:/root/.local/bin"')
package_file_name = module_manager.get_installed_packages_by_module(
GUI_PRODUCT_INSTALL_ID)[0].get_file_name()
with zipfile.ZipFile(Path.home() / ".lean" / "modules" /
package_file_name) as package_file:
content_file_names = [
f.replace("content/", "") for f in package_file.namelist()
if f.startswith("content/")
]
wheel_file_name = next(f for f in content_file_names
if f.endswith(".whl"))
terminal_file_name = next(f for f in content_file_names
if f.endswith(".zip"))
# Install the CLI in the GUI container
run_options["commands"].append("pip uninstall -y lean")
if lean.__version__ == "dev":
lean_cli_dir = str(
Path(__file__).absolute().parent.parent.parent.parent)
logger.info(
f"Detected lean dev version. Will mount local folder '{lean_cli_dir}' as /lean-cli"
)
run_options["volumes"][str(lean_cli_dir)] = {
"bind": "/lean-cli",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
else:
run_options["commands"].append(
"pip install --user --progress-bar off --upgrade lean")
# Install the GUI in the GUI container
run_options["commands"].append("pip uninstall -y leangui")
if gui is None:
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{wheel_file_name} > /{wheel_file_name}"
)
run_options["commands"].append(
f"pip install --user --progress-bar off /{wheel_file_name}")
elif gui.is_file():
run_options["mounts"].append(
Mount(target=f"/{gui.name}",
source=str(gui),
type="bind",
read_only=True))
run_options["commands"].append(
f"pip install --user --progress-bar off /{gui.name}")
else:
run_options["volumes"][str(gui)] = {
"bind": "/lean-cli-gui",
"mode": "rw"
}
run_options["commands"].append("cd /lean-cli-gui")
run_options["commands"].append(
"pip install --user --progress-bar off -r requirements.txt")
# Extract the terminal in the GUI container
run_options["commands"].append(
f"unzip -p /root/.lean/modules/{package_file_name} content/{terminal_file_name} > /{terminal_file_name}"
)
run_options["commands"].append(
f"unzip -o /{terminal_file_name} -d /terminal")
# Write correct streaming url to /terminal/local.socket.host.conf
run_options["commands"].append(
f'echo "ws://localhost:{port}/streaming" > /terminal/local.socket.host.conf'
)
# Mount the `lean init` directory in the GUI container
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
run_options["volumes"][str(cli_root_dir)] = {
"bind": "/LeanCLI",
"mode": "rw"
}
# Mount the global config directory in the GUI container
run_options["volumes"][str(Path("~/.lean").expanduser())] = {
"bind": "/root/.lean",
"mode": "rw"
}
# Mount a directory to the tmp directory in the GUI container
gui_tmp_directory = temp_manager.create_temporary_directory()
run_options["volumes"][str(gui_tmp_directory)] = {
"bind": "/tmp",
"mode": "rw"
}
# Set up the path mappings between paths in the host system and paths in the GUI container
run_options["environment"]["DOCKER_PATH_MAPPINGS"] = json.dumps({
"/LeanCLI":
cli_root_dir.as_posix(),
"/root/.lean":
Path("~/.lean").expanduser().as_posix(),
"/tmp":
gui_tmp_directory.as_posix()
})
# Mount the Docker socket in the GUI container
run_options["mounts"].append(
Mount(target="/var/run/docker.sock",
source="/var/run/docker.sock",
type="bind",
read_only=False))
# Run the GUI in the GUI container
run_options["commands"].append("cd /LeanCLI")
run_options["commands"].append(f"leangui")
# Don't delete temporary directories when the command exits, the container will still need them
temp_manager.delete_temporary_directories_when_done = False
logger.info("Starting the local GUI, this may take some time...")
# Pull the Docker images used by the local GUI
# If this is done while the local GUI is running there is a big delay between pressing Backtest and seeing it run
update_manager = container.update_manager()
cli_config_manager = container.cli_config_manager()
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_engine_image(), False)
update_manager.pull_docker_image_if_necessary(
cli_config_manager.get_research_image(), False)
try:
docker_manager.run_image(
DockerImage(name="python", tag="3.9.6-buster"), **run_options)
except APIError as error:
msg = error.explanation
if isinstance(msg, str) and any(m in msg.lower() for m in [
"port is already allocated", "ports are not available"
"an attempt was made to access a socket in a way forbidden by its access permissions"
]):
_error(
f"Port {port} is already in use, please specify a different port using --port <number>",
shortcut_launch)
raise error
url = f"http://localhost:{port}/"
# Wait until the GUI is running
while True:
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
if shortcut_launch:
_error(
"Something went wrong while starting the local GUI, run `lean gui logs` for more information",
shortcut_launch)
else:
_error(
"Something went wrong while starting the local GUI, see the logs above for more information",
shortcut_launch)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
logger.info(f"The local GUI has started and is running on {url}")
if not no_open:
webbrowser.open(url)
