def create_project(name: str, language: str) -> None: """Create a new project containing starter code. If NAME is a path containing subdirectories those will be created automatically. The default language can be set using `lean config set default-language python/csharp`. """ cli_config_manager = container.cli_config_manager() language = language if language is not None else cli_config_manager.default_language.get_value( ) if language is None: raise MoreInfoError( "Please specify a language with --language or set the default language using `lean config set default-language python/csharp`", "https://www.quantconnect.com/docs/v2/lean-cli/tutorials/project-management" ) full_path = Path.cwd() / name path_validator = container.path_validator() if not path_validator.is_path_valid(full_path): raise MoreInfoError( f"'{name}' is not a valid path", "https://www.quantconnect.com/docs/v2/lean-cli/user-guides/troubleshooting#02-Common-errors" ) if full_path.exists(): raise RuntimeError( f"A project named '{name}' already exists, please choose a different name" ) else: project_manager = container.project_manager() project_manager.create_new_project( full_path, QCLanguage.Python if language == "python" else QCLanguage.CSharp) # Convert the project name into a valid class name by removing all non-alphanumeric characters class_name = re.sub(f"[^a-zA-Z0-9]", "", "".join(map(_capitalize, name.split(" ")))) if language == "python": with (full_path / "main.py").open("w+", encoding="utf-8") as file: file.write(DEFAULT_PYTHON_MAIN.replace("$NAME$", class_name)) else: with (full_path / "Main.cs").open("w+", encoding="utf-8") as file: file.write(DEFAULT_CSHARP_MAIN.replace("$NAME$", class_name)) with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file: file.write(DEFAULT_PYTHON_NOTEBOOK if language == "python" else DEFAULT_CSHARP_NOTEBOOK) with (full_path / "config.json").open("w+", encoding="utf-8") as file: file.write(DEFAULT_PYTHON_CONFIG if language == "python" else DEFAULT_CSHARP_CONFIG) logger = container.logger() logger.info( f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'" )
def live(project: Path, environment: str, output: Optional[Path], image: Optional[str], update: bool) -> None: """Start live trading a project locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b ENVIRONMENT must be the name of an environment in the Lean configuration file with live-mode set to true. By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(Path(project)) if output is None: output = algorithm_file.parent / "live" / datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") lean_config_manager = container.lean_config_manager() lean_config = lean_config_manager.get_complete_lean_config( environment, algorithm_file, None) if "environments" not in lean_config or environment not in lean_config[ "environments"]: lean_config_path = lean_config_manager.get_lean_config_path() raise MoreInfoError( f"{lean_config_path} does not contain an environment named '{environment}'", "https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading" ) if not lean_config["environments"][environment]["live-mode"]: raise MoreInfoError( f"The '{environment}' is not a live trading environment (live-mode is set to false)", "https://www.quantconnect.com/docs/v2/lean-cli/tutorials/live-trading/local-live-trading" ) _raise_for_missing_properties(lean_config, environment, lean_config_manager.get_lean_config_path()) _start_iqconnect_if_necessary(lean_config, environment) cli_config_manager = container.cli_config_manager() engine_image = cli_config_manager.get_engine_image(image) docker_manager = container.docker_manager() if update or not docker_manager.supports_dotnet_5(engine_image): docker_manager.pull_image(engine_image) lean_runner = container.lean_runner() lean_runner.run_lean(environment, algorithm_file, output, engine_image, None) if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update: update_manager = container.update_manager() update_manager.warn_if_docker_image_outdated(engine_image)
def create_project(name: str, language: str) -> None: """Create a new project containing starter code. If NAME is a path containing subdirectories those will be created automatically. The default language can be set using `lean config set default-language python/csharp`. """ cli_config_manager = container.cli_config_manager() language = language if language is not None else cli_config_manager.default_language.get_value() if language is None: raise MoreInfoError( "Please specify a language with --language or set the default language using `lean config set default-language python/csharp`", "https://www.lean.io/docs/lean-cli/projects/project-management") full_path = Path.cwd() / name if not container.path_manager().is_path_valid(full_path): raise MoreInfoError(f"'{name}' is not a valid path", "https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors") is_library_project = False try: library_dir = container.lean_config_manager().get_cli_root_directory() / "Library" is_library_project = library_dir in full_path.parents except: # get_cli_root_directory() raises an error if there is no such directory pass if is_library_project and language == "python" and not full_path.name.isidentifier(): raise RuntimeError( f"'{full_path.name}' is not a valid Python identifier, which is required for Python library projects to be importable") if full_path.exists(): raise RuntimeError(f"A project named '{name}' already exists, please choose a different name") else: project_manager = container.project_manager() project_manager.create_new_project(full_path, QCLanguage.Python if language == "python" else QCLanguage.CSharp) # Convert the project name into a valid class name by removing all non-alphanumeric characters class_name = re.sub(f"[^a-zA-Z0-9]", "", "".join(map(_capitalize, full_path.name.split(" ")))) if language == "python": main_name = "main.py" main_content = DEFAULT_PYTHON_MAIN if not is_library_project else LIBRARY_PYTHON_MAIN else: main_name = "Main.cs" main_content = DEFAULT_CSHARP_MAIN if not is_library_project else LIBRARY_CSHARP_MAIN with (full_path / main_name).open("w+", encoding="utf-8") as file: file.write(main_content.replace("$CLASS_NAME$", class_name).replace("$PROJECT_NAME$", full_path.name)) with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file: file.write(DEFAULT_PYTHON_NOTEBOOK if language == "python" else DEFAULT_CSHARP_NOTEBOOK) logger = container.logger() logger.info(f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'")
def _migrate_dotnet_5_csharp_rider(project_dir: Path) -> None: made_changes = False for dir_name in [ f".idea.{project_dir.stem}", f".idea.{project_dir.stem}.dir" ]: workspace_xml_path = project_dir / ".idea" / dir_name / ".idea" / "workspace.xml" if not workspace_xml_path.is_file(): continue current_content = ElementTree.fromstring( workspace_xml_path.read_text(encoding="utf-8")) run_manager = current_content.find(".//component[@name='RunManager']") if run_manager is None: continue config = run_manager.find( ".//configuration[@name='Debug with Lean CLI']") if config is None: continue run_manager.remove(config) new_content = ElementTree.tostring(current_content, encoding="utf-8", method="xml").decode("utf-8") workspace_xml_path.write_text(new_content, encoding="utf-8") made_changes = True if made_changes: container.project_manager().generate_rider_config() logger = container.logger() logger.warn( "Your run configuration has been updated to work with the .NET 5 version of LEAN" ) logger.warn("Please restart Rider and start debugging again") logger.warn( "See https://www.quantconnect.com/docs/v2/lean-cli/tutorials/backtesting/debugging-local-backtests#05-C-and-Rider for the updated instructions" ) raise click.Abort()
def _migrate_csharp_rider(project_dir: Path) -> None: made_changes = False xml_manager = container.xml_manager() for dir_name in [ f".idea.{project_dir.stem}", f".idea.{project_dir.stem}.dir" ]: workspace_xml_path = project_dir / ".idea" / dir_name / ".idea" / "workspace.xml" if not workspace_xml_path.is_file(): continue current_content = xml_manager.parse( workspace_xml_path.read_text(encoding="utf-8")) run_manager = current_content.find(".//component[@name='RunManager']") if run_manager is None: continue config = run_manager.find( ".//configuration[@name='Debug with Lean CLI']") if config is None: continue run_manager.remove(config) workspace_xml_path.write_text(xml_manager.to_string(current_content), encoding="utf-8") made_changes = True if made_changes: container.project_manager().generate_rider_config() logger = container.logger() logger.warn( "Your run configuration has been updated to work with the .NET 5 version of LEAN" ) logger.warn("Please restart Rider and start debugging again") logger.warn( "See https://www.lean.io/docs/lean-cli/backtesting/debugging#05-C-and-Rider for the updated instructions" ) raise click.Abort()
def backtest(project: Path, output: Optional[Path], debug: Optional[str], image: Optional[str], update: bool) -> None: """Backtest a project locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b Go to the following url to learn how to debug backtests locally using the Lean CLI: https://www.quantconnect.com/docs/v2/lean-cli/tutorials/backtesting/debugging-local-backtests By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(Path(project)) if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") debugging_method = None if debug == "pycharm": debugging_method = DebuggingMethod.PyCharm _migrate_dotnet_5_python_pycharm(algorithm_file.parent) elif debug == "ptvsd": debugging_method = DebuggingMethod.PTVSD _migrate_dotnet_5_python_vscode(algorithm_file.parent) elif debug == "vsdbg": debugging_method = DebuggingMethod.VSDBG _migrate_dotnet_5_csharp_vscode(algorithm_file.parent) elif debug == "rider": debugging_method = DebuggingMethod.Rider _migrate_dotnet_5_csharp_rider(algorithm_file.parent) cli_config_manager = container.cli_config_manager() engine_image = cli_config_manager.get_engine_image(image) docker_manager = container.docker_manager() if update or not docker_manager.supports_dotnet_5(engine_image): docker_manager.pull_image(engine_image) lean_runner = container.lean_runner() lean_runner.run_lean("backtesting", algorithm_file, output, engine_image, debugging_method) if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update: update_manager = container.update_manager() update_manager.warn_if_docker_image_outdated(engine_image)
def backtest(project: Path, output: Optional[Path], update: bool, version: Optional[int], debug: Optional[str]) -> None: """Backtest a project locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b Go to the following url to learn how to debug backtests using the Lean CLI: https://github.com/QuantConnect/lean-cli#debugging-backtests """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(Path(project)) if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") lean_runner = container.lean_runner() if update: lean_runner.force_update() # Detect the debugging method to use based on the editor and project language debugging_method = None if debug == "pycharm": debugging_method = "PyCharm" if debug == "vs": debugging_method = "VisualStudio" if debug == "vscode": debugging_method = "PTVSD" if algorithm_file.name.endswith( ".py") else "VisualStudio" lean_runner.run_lean("backtesting", algorithm_file, output, version, debugging_method)
def optimize(project: Path, output: Optional[Path], optimizer_config: Optional[Path], image: Optional[str], update: bool) -> None: """Optimize a project's parameters locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b The --optimizer-config option can be used to specify the configuration to run the optimizer with. When using the option it should point to a file like this (the algorithm-* properties should be omitted): https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json When --optimizer-config is not set, an interactive prompt will be shown to configure the optimizer. By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(project) if output is None: output = algorithm_file.parent / "optimizations" / datetime.now( ).strftime("%Y-%m-%d_%H-%M-%S") if optimizer_config is None: project_config_manager = container.project_config_manager() project_config = project_config_manager.get_project_config( algorithm_file.parent) project_parameters = [ QCParameter(key=k, value=v) for k, v in project_config.get("parameters", {}).items() ] if len(project_parameters) == 0: raise MoreInfoError( "The given project has no parameters to optimize", "https://www.lean.io/docs/lean-cli/tutorials/optimization/project-parameters" ) optimizer_config_manager = container.optimizer_config_manager() optimization_strategy = optimizer_config_manager.configure_strategy( cloud=False) optimization_target = optimizer_config_manager.configure_target() optimization_parameters = optimizer_config_manager.configure_parameters( project_parameters, cloud=False) optimization_constraints = optimizer_config_manager.configure_constraints( ) config = { "optimization-strategy": optimization_strategy, "optimization-strategy-settings": { "$type": "QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer", "default-segment-amount": 10 }, "optimization-criterion": { "target": optimization_target.target, "extremum": optimization_target.extremum.value }, "parameters": [parameter.dict() for parameter in optimization_parameters], "constraints": [ constraint.dict(by_alias=True) for constraint in optimization_constraints ] } else: config = json5.loads(optimizer_config.read_text(encoding="utf-8")) # Remove keys which are configured in the Lean config for key in [ "algorithm-type-name", "algorithm-language", "algorithm-location" ]: config.pop(key, None) config["optimizer-close-automatically"] = True config["results-destination-folder"] = "/Results" config_path = output / "optimizer-config.json" config_path.parent.mkdir(parents=True, exist_ok=True) with config_path.open("w+", encoding="utf-8") as file: file.write(json.dumps(config, indent=4) + "\n") cli_config_manager = container.cli_config_manager() engine_image = cli_config_manager.get_engine_image(image) lean_config_manager = container.lean_config_manager() lean_config = lean_config_manager.get_complete_lean_config( "backtesting", algorithm_file, None, None) lean_runner = container.lean_runner() run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None) run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug" run_options["commands"].append( "dotnet QuantConnect.Optimizer.Launcher.dll") run_options["mounts"].append( Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json", source=str(config_path), type="bind", read_only=True)) docker_manager = container.docker_manager() if update or not docker_manager.supports_dotnet_5(engine_image): docker_manager.pull_image(engine_image) success = docker_manager.run_image(engine_image, **run_options) cli_root_dir = container.lean_config_manager().get_cli_root_directory() relative_project_dir = project.relative_to(cli_root_dir) relative_output_dir = output.relative_to(cli_root_dir) if success: logger = container.logger() optimizer_logs = (output / "log.txt").read_text(encoding="utf-8") groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'", optimizer_logs) if len(groups) > 0: optimal_parameters, optimal_id = groups[0] optimal_results = json.loads( (output / optimal_id / f"{optimal_id}.json").read_text(encoding="utf-8")) optimal_backtest = QCBacktest( backtestId=optimal_id, projectId=1, status="", name=optimal_id, created=datetime.now(), completed=True, progress=1.0, runtimeStatistics=optimal_results["RuntimeStatistics"], statistics=optimal_results["Statistics"]) logger.info( f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}" ) logger.info(f"Optimal backtest results:") logger.info(optimal_backtest.get_statistics_table()) logger.info( f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'" ) else: raise RuntimeError( f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'" ) if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update: update_manager = container.update_manager() update_manager.warn_if_docker_image_outdated(engine_image)
def research(project: Path, port: int, data_provider: Optional[str], download_data: bool, data_purchase_limit: Optional[int], detach: bool, no_open: bool, image: Optional[str], update: bool) -> None: """Run a Jupyter Lab environment locally using Docker. By default the official LEAN research image is used. You can override this using the --image option. Alternatively you can set the default research image using `lean config set research-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(project) lean_config_manager = container.lean_config_manager() lean_config = lean_config_manager.get_complete_lean_config( "backtesting", algorithm_file, None) lean_config["composer-dll-directory"] = "/Lean/Launcher/bin/Debug" if download_data: data_provider = QuantConnectDataProvider.get_name() if data_provider is not None: data_provider = next(dp for dp in all_data_providers if dp.get_name() == data_provider) data_provider.build(lean_config, container.logger()).configure( lean_config, "backtesting") lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) lean_runner = container.lean_runner() temp_manager = container.temp_manager() run_options = lean_runner.get_basic_docker_config( lean_config, algorithm_file, temp_manager.create_temporary_directory(), None, False, detach) # Mount the config in the notebooks directory as well local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json")) run_options["mounts"].append( Mount(target="/Lean/Launcher/bin/Debug/Notebooks/config.json", source=str(local_config_path), type="bind", read_only=True)) # Jupyter Lab runs on port 8888, we expose it to the local port specified by the user run_options["ports"]["8888"] = str(port) # Open the browser as soon as Jupyter Lab has started if detach or not no_open: run_options["on_output"] = lambda chunk: _check_docker_output( chunk, port) # Give container an identifiable name when running it from the GUI if container.module_manager().is_module_installed(GUI_PRODUCT_INSTALL_ID): project_id = container.project_config_manager().get_local_id( algorithm_file.parent) run_options["name"] = f"lean_cli_gui_research_{project_id}" # Make Ctrl+C stop Jupyter Lab immediately run_options["stop_signal"] = "SIGKILL" # Mount the project to the notebooks directory run_options["volumes"][str(project)] = { "bind": "/Lean/Launcher/bin/Debug/Notebooks", "mode": "rw" } # Add references to all DLLs in QuantConnect.csx so custom C# libraries can be imported with using statements run_options["commands"].append(" && ".join([ 'find . -maxdepth 1 -iname "*.dll" | xargs -I _ echo \'#r "_"\' | cat - QuantConnect.csx > NewQuantConnect.csx', "mv NewQuantConnect.csx QuantConnect.csx" ])) # Allow notebooks to be embedded in iframes run_options["commands"].append("mkdir -p ~/.jupyter") run_options["commands"].append( 'echo "c.NotebookApp.disable_check_xsrf = True\nc.NotebookApp.tornado_settings = {\'headers\': {\'Content-Security-Policy\': \'frame-ancestors self *\'}}" > ~/.jupyter/jupyter_notebook_config.py' ) # Hide headers in notebooks run_options["commands"].append( "mkdir -p ~/.ipython/profile_default/static/custom") run_options["commands"].append( 'echo "#header-container { display: none !important; }" > ~/.ipython/profile_default/static/custom/custom.css' ) # Run the script that starts Jupyter Lab when all set up has been done run_options["commands"].append("./start.sh") project_config_manager = container.project_config_manager() cli_config_manager = container.cli_config_manager() project_config = project_config_manager.get_project_config( algorithm_file.parent) research_image = cli_config_manager.get_research_image( image or project_config.get("research-image", None)) container.update_manager().pull_docker_image_if_necessary( research_image, update) try: container.docker_manager().run_image(research_image, **run_options) except APIError as error: msg = error.explanation if isinstance(msg, str) and any(m in msg.lower() for m in [ "port is already allocated", "ports are not available" "an attempt was made to access a socket in a way forbidden by its access permissions" ]): raise RuntimeError( f"Port {port} is already in use, please specify a different port using --port <number>" ) raise error if detach: temp_manager.delete_temporary_directories_when_done = False logger = container.logger() relative_project_dir = algorithm_file.parent.relative_to( lean_config_manager.get_cli_root_directory()) logger.info( f"Successfully started Jupyter Lab environment for '{relative_project_dir}' in the '{run_options['name']}' container" ) logger.info( "You can use Docker's own commands to manage the detached container" )
def backtest(project: Path, output: Optional[Path], detach: bool, debug: Optional[str], data_provider: Optional[str], download_data: bool, data_purchase_limit: Optional[int], release: bool, image: Optional[str], update: bool) -> None: """Backtest a project locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b Go to the following url to learn how to debug backtests locally using the Lean CLI: https://www.lean.io/docs/lean-cli/tutorials/backtesting/debugging By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(Path(project)) lean_config_manager = container.lean_config_manager() if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") debugging_method = None if debug == "pycharm": debugging_method = DebuggingMethod.PyCharm _migrate_python_pycharm(algorithm_file.parent) elif debug == "ptvsd": debugging_method = DebuggingMethod.PTVSD _migrate_python_vscode(algorithm_file.parent) elif debug == "vsdbg": debugging_method = DebuggingMethod.VSDBG _migrate_csharp_vscode(algorithm_file.parent) elif debug == "rider": debugging_method = DebuggingMethod.Rider _migrate_csharp_rider(algorithm_file.parent) if debugging_method is not None and detach: raise RuntimeError( "Running a debugging session in a detached container is not supported" ) if algorithm_file.name.endswith(".cs"): _migrate_csharp_csproj(algorithm_file.parent) lean_config = lean_config_manager.get_complete_lean_config( "backtesting", algorithm_file, debugging_method) if download_data: data_provider = QuantConnectDataProvider.get_name() if data_provider is not None: data_provider = next(dp for dp in all_data_providers if dp.get_name() == data_provider) data_provider.build(lean_config, container.logger()).configure( lean_config, "backtesting") lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) cli_config_manager = container.cli_config_manager() project_config_manager = container.project_config_manager() project_config = project_config_manager.get_project_config( algorithm_file.parent) engine_image = cli_config_manager.get_engine_image( image or project_config.get("engine-image", None)) container.update_manager().pull_docker_image_if_necessary( engine_image, update) if not output.exists(): output.mkdir(parents=True) output_config_manager = container.output_config_manager() lean_config["algorithm-id"] = str( output_config_manager.get_backtest_id(output)) lean_runner = container.lean_runner() lean_runner.run_lean(lean_config, "backtesting", algorithm_file, output, engine_image, debugging_method, release, detach)
def backtest(project: Path, output: Optional[Path], debug: Optional[str], download_data: bool, data_purchase_limit: Optional[int], image: Optional[str], update: bool) -> None: """Backtest a project locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. \b Go to the following url to learn how to debug backtests locally using the Lean CLI: https://www.lean.io/docs/lean-cli/tutorials/backtesting/debugging-local-backtests By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(Path(project)) lean_config_manager = container.lean_config_manager() if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") debugging_method = None if debug == "pycharm": debugging_method = DebuggingMethod.PyCharm _migrate_dotnet_5_python_pycharm(algorithm_file.parent) elif debug == "ptvsd": debugging_method = DebuggingMethod.PTVSD _migrate_dotnet_5_python_vscode(algorithm_file.parent) elif debug == "vsdbg": debugging_method = DebuggingMethod.VSDBG _migrate_dotnet_5_csharp_vscode(algorithm_file.parent) elif debug == "rider": debugging_method = DebuggingMethod.Rider _migrate_dotnet_5_csharp_rider(algorithm_file.parent) if download_data: organization = _select_organization() lean_config_manager.set_property("job-organization-id", organization.id) lean_config_manager.set_property( "data-provider", "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider") lean_config_manager.set_property( "map-file-provider", "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider") lean_config_manager.set_property( "factor-file-provider", "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider") if data_purchase_limit is not None: config = lean_config_manager.get_lean_config() if config.get( "data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": container.logger().warn( "--data-purchase-limit is ignored because the data provider is not set to download from the API, use --download-data to set that up" ) data_purchase_limit = None cli_config_manager = container.cli_config_manager() engine_image = cli_config_manager.get_engine_image(image) docker_manager = container.docker_manager() if update or not docker_manager.supports_dotnet_5(engine_image): docker_manager.pull_image(engine_image) lean_config = lean_config_manager.get_complete_lean_config( "backtesting", algorithm_file, debugging_method, data_purchase_limit) lean_runner = container.lean_runner() lean_runner.run_lean(lean_config, "backtesting", algorithm_file, output, engine_image, debugging_method) if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update: update_manager = container.update_manager() update_manager.warn_if_docker_image_outdated(engine_image)
def optimize(project: Path, output: Optional[Path], detach: bool, optimizer_config: Optional[Path], strategy: Optional[str], target: Optional[str], target_direction: Optional[str], parameter: List[Tuple[str, float, float, float]], constraint: List[str], release: bool, image: Optional[str], update: bool) -> None: """Optimize a project's parameters locally using Docker. \b If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed. If PROJECT is a file, the algorithm in the specified file will be executed. By default an interactive wizard is shown letting you configure the optimizer. If --optimizer-config or --strategy is given the command runs in non-interactive mode. In this mode the CLI does not prompt for input. \b The --optimizer-config option can be used to specify the configuration to run the optimizer with. When using the option it should point to a file like this (the algorithm-* properties should be omitted): https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json If --strategy is given the optimizer configuration is read from the given options. In this case --strategy, --target, --target-direction and --parameter become required. \b In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters: - --parameter <name> <min value> <max value> <step size> - --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5 \b In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints: - --constraint "<statistic> <operator> <value>" - --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25" By default the official LEAN engine image is used. You can override this using the --image option. Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`. """ project_manager = container.project_manager() algorithm_file = project_manager.find_algorithm_file(project) if output is None: output = algorithm_file.parent / "optimizations" / datetime.now( ).strftime("%Y-%m-%d_%H-%M-%S") optimizer_config_manager = container.optimizer_config_manager() config = None if optimizer_config is not None and strategy is not None: raise RuntimeError( "--optimizer-config and --strategy are mutually exclusive") if optimizer_config is not None: config = json5.loads(optimizer_config.read_text(encoding="utf-8")) # Remove keys which are configured in the Lean config for key in [ "algorithm-type-name", "algorithm-language", "algorithm-location" ]: config.pop(key, None) elif strategy is not None: ensure_options(["strategy", "target", "target_direction", "parameter"]) optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy" optimization_target = OptimizationTarget( target=optimizer_config_manager.parse_target(target), extremum=target_direction) optimization_parameters = optimizer_config_manager.parse_parameters( parameter) optimization_constraints = optimizer_config_manager.parse_constraints( constraint) else: project_config_manager = container.project_config_manager() project_config = project_config_manager.get_project_config( algorithm_file.parent) project_parameters = [ QCParameter(key=k, value=v) for k, v in project_config.get("parameters", {}).items() ] if len(project_parameters) == 0: raise MoreInfoError( "The given project has no parameters to optimize", "https://www.lean.io/docs/lean-cli/optimization/parameters") optimization_strategy = optimizer_config_manager.configure_strategy( cloud=False) optimization_target = optimizer_config_manager.configure_target() optimization_parameters = optimizer_config_manager.configure_parameters( project_parameters, cloud=False) optimization_constraints = optimizer_config_manager.configure_constraints( ) if config is None: # noinspection PyUnboundLocalVariable config = { "optimization-strategy": optimization_strategy, "optimization-strategy-settings": { "$type": "QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer", "default-segment-amount": 10 }, "optimization-criterion": { "target": optimization_target.target, "extremum": optimization_target.extremum.value }, "parameters": [parameter.dict() for parameter in optimization_parameters], "constraints": [ constraint.dict(by_alias=True) for constraint in optimization_constraints ] } config["optimizer-close-automatically"] = True config["results-destination-folder"] = "/Results" config_path = output / "optimizer-config.json" config_path.parent.mkdir(parents=True, exist_ok=True) with config_path.open("w+", encoding="utf-8") as file: file.write(json.dumps(config, indent=4) + "\n") project_config_manager = container.project_config_manager() cli_config_manager = container.cli_config_manager() project_config = project_config_manager.get_project_config( algorithm_file.parent) engine_image = cli_config_manager.get_engine_image( image or project_config.get("engine-image", None)) lean_config_manager = container.lean_config_manager() lean_config = lean_config_manager.get_complete_lean_config( "backtesting", algorithm_file, None) if not output.exists(): output.mkdir(parents=True) output_config_manager = container.output_config_manager() lean_config["algorithm-id"] = str( output_config_manager.get_optimization_id(output)) lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging" lean_runner = container.lean_runner() run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, detach) run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug" run_options["commands"].append( "dotnet QuantConnect.Optimizer.Launcher.dll") run_options["mounts"].append( Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json", source=str(config_path), type="bind", read_only=True)) container.update_manager().pull_docker_image_if_necessary( engine_image, update) project_manager.copy_code(algorithm_file.parent, output / "code") success = container.docker_manager().run_image(engine_image, **run_options) logger = container.logger() cli_root_dir = container.lean_config_manager().get_cli_root_directory() relative_project_dir = project.relative_to(cli_root_dir) relative_output_dir = output.relative_to(cli_root_dir) if detach: temp_manager = container.temp_manager() temp_manager.delete_temporary_directories_when_done = False logger.info( f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container" ) logger.info(f"The output will be stored in '{relative_output_dir}'") logger.info( "You can use Docker's own commands to manage the detached container" ) elif success: optimizer_logs = (output / "log.txt").read_text(encoding="utf-8") groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'", optimizer_logs) if len(groups) > 0: optimal_parameters, optimal_id = groups[0] optimal_results = json.loads( (output / optimal_id / f"{optimal_id}.json").read_text(encoding="utf-8")) optimal_backtest = QCBacktest( backtestId=optimal_id, projectId=1, status="", name=optimal_id, created=datetime.now(), completed=True, progress=1.0, runtimeStatistics=optimal_results["RuntimeStatistics"], statistics=optimal_results["Statistics"]) logger.info( f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}" ) logger.info(f"Optimal backtest results:") logger.info(optimal_backtest.get_statistics_table()) logger.info( f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'" ) else: raise RuntimeError( f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'" )