def test_cloud_optimize_passes_given_config_to_cloud_runner() -> None:
create_fake_lean_cli_directory()
project = create_api_project(1, "My Project")
optimization = create_api_optimization()
api_client = mock.Mock()
api_client.projects.get_all.return_value = [project]
api_client.optimizations.estimate.return_value = QCOptimizationEstimate(
estimateId="x", time=10, balance=1000)
api_client.organizations.get.return_value = create_api_organization()
container.api_client.override(providers.Object(api_client))
cloud_runner = mock.Mock()
cloud_runner.run_optimization.return_value = optimization
container.cloud_runner.override(providers.Object(cloud_runner))
result = CliRunner().invoke(
lean, ["cloud", "optimize", "My Project", "--name", "My Name"])
assert result.exit_code == 0
optimizer_config_manager = container.optimizer_config_manager()
cloud_runner.run_optimization.assert_called_once_with(
project, mock.ANY, "My Name",
optimizer_config_manager.configure_strategy(cloud=True),
optimizer_config_manager.configure_target(),
optimizer_config_manager.configure_parameters([]),
optimizer_config_manager.configure_constraints(),
optimizer_config_manager.configure_node()[0].name,
optimizer_config_manager.configure_node()[1])
def _backtest_meets_constraints(backtest: QCOptimizationBacktest, constraints: List[OptimizationConstraint]) -> bool:
"""Returns whether the backtest meets all constraints.
:param backtest: the backtest to check
:param constraints: the constraints the backtest has to meet
:return: True if the backtest meets all constraints, False if not
"""
optimizer_config_manager = container.optimizer_config_manager()
for constraint in constraints:
expression = str(constraint)
for target, _ in optimizer_config_manager.available_targets:
expression = expression.replace(target, str(_get_backtest_statistic(backtest, target)))
if not eval(expression):
return False
return True
def optimize(project: Path, output: Optional[Path],
optimizer_config: Optional[Path], image: Optional[str],
update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
When --optimizer-config is not set, an interactive prompt will be shown to configure the optimizer.
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
if optimizer_config is None:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/tutorials/optimization/project-parameters"
)
optimizer_config_manager = container.optimizer_config_manager()
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
else:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
cli_config_manager = container.cli_config_manager()
engine_image = cli_config_manager.get_engine_image(image)
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None, None)
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
docker_manager = container.docker_manager()
if update or not docker_manager.supports_dotnet_5(engine_image):
docker_manager.pull_image(engine_image)
success = docker_manager.run_image(engine_image, **run_options)
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if success:
logger = container.logger()
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)
if str(engine_image) == DEFAULT_ENGINE_IMAGE and not update:
update_manager = container.update_manager()
update_manager.warn_if_docker_image_outdated(engine_image)
def optimize(project: str, target: Optional[str],
target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
node: Optional[str], parallel_nodes: Optional[int],
name: Optional[str], push: bool) -> None:
"""Optimize a project in the cloud.
PROJECT must be the name or id of the project to optimize.
An interactive prompt will be shown to configure the optimizer.
If --target is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input and the following options become required:
--target, --target-direction, --parameter, --node and --parallel-nodes.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
If the project that has to be optimized has been pulled to the local drive
with `lean cloud pull` it is possible to use the --push option to push local
modifications to the cloud before running the optimization.
"""
logger = container.logger()
api_client = container.api_client()
cloud_project_manager = container.cloud_project_manager()
cloud_project = cloud_project_manager.get_cloud_project(project, push)
if name is None:
name = container.name_generator().generate_name()
cloud_runner = container.cloud_runner()
finished_compile = cloud_runner.compile_project(cloud_project)
optimizer_config_manager = container.optimizer_config_manager()
organization = api_client.organizations.get(cloud_project.organizationId)
if target is not None:
ensure_options([
"target", "target_direction", "parameter", "node", "parallel_nodes"
])
optimization_strategy = "QuantConnect.Optimizer.Strategies.GridSearchOptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
node = next(n for n in available_nodes if n.name == node)
if parallel_nodes < node.min_nodes:
raise RuntimeError(
f"The minimum number of parallel nodes for {node.name} is {node.min_nodes}"
)
if parallel_nodes > node.max_nodes:
raise RuntimeError(
f"The maximum number of parallel nodes for {node.name} is {node.max_nodes}"
)
_display_estimate(cloud_project, finished_compile, organization, name,
optimization_strategy, optimization_target,
optimization_parameters, optimization_constraints,
node, parallel_nodes)
else:
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=True)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
cloud_project.parameters, cloud=True)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
while True:
node, parallel_nodes = optimizer_config_manager.configure_node()
_display_estimate(cloud_project, finished_compile, organization,
name, optimization_strategy, optimization_target,
optimization_parameters,
optimization_constraints, node, parallel_nodes)
if click.confirm(
"Do you want to start the optimization on the selected node type?",
default=True):
break
optimization = cloud_runner.run_optimization(
cloud_project, finished_compile, name, optimization_strategy,
optimization_target, optimization_parameters, optimization_constraints,
node.name, parallel_nodes)
backtests = optimization.backtests.values()
backtests = [b for b in backtests if b.exitCode == 0]
backtests = [
b for b in backtests
if _backtest_meets_constraints(b, optimization_constraints)
]
if len(backtests) == 0:
logger.info(
"No optimal parameter combination found, no successful backtests meet all constraints"
)
return
optimal_backtest = sorted(backtests,
key=lambda backtest: _get_backtest_statistic(
backtest, optimization_target.target),
reverse=optimization_target.extremum ==
OptimizationExtremum.Maximum)[0]
parameters = ", ".join(f"{key}: {optimal_backtest.parameterSet[key]}"
for key in optimal_backtest.parameterSet)
logger.info(f"Optimal parameters: {parameters}")
optimal_backtest = api_client.backtests.get(cloud_project.projectId,
optimal_backtest.id)
logger.info(f"Optimal backtest id: {optimal_backtest.backtestId}")
logger.info(f"Optimal backtest name: {optimal_backtest.name}")
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
def optimize(project: str, name: Optional[str], push: bool) -> None:
"""Optimize a project in the cloud.
An interactive prompt will be shown to configure the optimizer.
PROJECT must be the name or id of the project to optimize.
If the project that has to be optimized has been pulled to the local drive
with `lean cloud pull` it is possible to use the --push option to push local
modifications to the cloud before running the optimization.
"""
logger = container.logger()
api_client = container.api_client()
cloud_project_manager = container.cloud_project_manager()
cloud_project = cloud_project_manager.get_cloud_project(project, push)
if name is None:
name = container.name_generator().generate_name()
cloud_runner = container.cloud_runner()
finished_compile = cloud_runner.compile_project(cloud_project)
optimizer_config_manager = container.optimizer_config_manager()
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=True)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
cloud_project.parameters, cloud=True)
optimization_constraints = optimizer_config_manager.configure_constraints()
backtest_count = _calculate_backtest_count(optimization_parameters)
organization = api_client.organizations.get(cloud_project.organizationId)
while True:
node, parallel_nodes = optimizer_config_manager.configure_node()
estimate = api_client.optimizations.estimate(
cloud_project.projectId, finished_compile.compileId, name,
optimization_strategy, optimization_target,
optimization_parameters, optimization_constraints, node.name,
parallel_nodes)
hours = _calculate_hours(estimate.time, backtest_count)
batch_time = ceil((hours * 100) / parallel_nodes) / 100
batch_cost = max(0.01, ceil(node.price * hours * 100) / 100)
logger.info(f"Estimated number of backtests: {backtest_count:,}")
logger.info(f"Estimated batch time: {_format_hours(batch_time)}")
logger.info(f"Estimated batch cost: ${batch_cost:,.2f}")
logger.info(
f"Organization balance: {organization.credit.balance:,.0f} QCC (${organization.credit.balance / 100:,.2f})"
)
if click.confirm(
"Do you want to start the optimization on the selected node type?",
default=True):
break
optimization = cloud_runner.run_optimization(
cloud_project, finished_compile, name, optimization_strategy,
optimization_target, optimization_parameters, optimization_constraints,
node.name, parallel_nodes)
backtests = optimization.backtests.values()
backtests = [b for b in backtests if b.exitCode == 0]
backtests = [
b for b in backtests
if _backtest_meets_constraints(b, optimization_constraints)
]
if len(backtests) == 0:
logger.info(
"No optimal parameter combination found, no successful backtests meet all constraints"
)
return
optimal_backtest = sorted(backtests,
key=lambda backtest: _get_backtest_statistic(
backtest, optimization_target.target),
reverse=optimization_target.extremum ==
OptimizationExtremum.Maximum)[0]
parameters = ", ".join(f"{key}: {optimal_backtest.parameterSet[key]}"
for key in optimal_backtest.parameterSet)
logger.info(f"Optimal parameters: {parameters}")
optimal_backtest = api_client.backtests.get(cloud_project.projectId,
optimal_backtest.id)
logger.info(f"Optimal backtest id: {optimal_backtest.backtestId}")
logger.info(f"Optimal backtest name: {optimal_backtest.name}")
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
def optimize(project: Path, output: Optional[Path], detach: bool,
optimizer_config: Optional[Path], strategy: Optional[str],
target: Optional[str], target_direction: Optional[str],
parameter: List[Tuple[str, float, float,
float]], constraint: List[str],
release: bool, image: Optional[str], update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
\b
If PROJECT is a directory, the algorithm in the main.py or Main.cs file inside it will be executed.
If PROJECT is a file, the algorithm in the specified file will be executed.
By default an interactive wizard is shown letting you configure the optimizer.
If --optimizer-config or --strategy is given the command runs in non-interactive mode.
In this mode the CLI does not prompt for input.
\b
The --optimizer-config option can be used to specify the configuration to run the optimizer with.
When using the option it should point to a file like this (the algorithm-* properties should be omitted):
https://github.com/QuantConnect/Lean/blob/master/Optimizer.Launcher/config.json
If --strategy is given the optimizer configuration is read from the given options.
In this case --strategy, --target, --target-direction and --parameter become required.
\b
In non-interactive mode the --parameter option can be provided multiple times to configure multiple parameters:
- --parameter <name> <min value> <max value> <step size>
- --parameter my-first-parameter 1 10 0.5 --parameter my-second-parameter 20 30 5
\b
In non-interactive mode the --constraint option can be provided multiple times to configure multiple constraints:
- --constraint "<statistic> <operator> <value>"
- --constraint "Sharpe Ratio >= 0.5" --constraint "Drawdown < 0.25"
By default the official LEAN engine image is used.
You can override this using the --image option.
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
project_manager = container.project_manager()
algorithm_file = project_manager.find_algorithm_file(project)
if output is None:
output = algorithm_file.parent / "optimizations" / datetime.now(
).strftime("%Y-%m-%d_%H-%M-%S")
optimizer_config_manager = container.optimizer_config_manager()
config = None
if optimizer_config is not None and strategy is not None:
raise RuntimeError(
"--optimizer-config and --strategy are mutually exclusive")
if optimizer_config is not None:
config = json5.loads(optimizer_config.read_text(encoding="utf-8"))
# Remove keys which are configured in the Lean config
for key in [
"algorithm-type-name", "algorithm-language",
"algorithm-location"
]:
config.pop(key, None)
elif strategy is not None:
ensure_options(["strategy", "target", "target_direction", "parameter"])
optimization_strategy = f"QuantConnect.Optimizer.Strategies.{strategy.replace(' ', '')}OptimizationStrategy"
optimization_target = OptimizationTarget(
target=optimizer_config_manager.parse_target(target),
extremum=target_direction)
optimization_parameters = optimizer_config_manager.parse_parameters(
parameter)
optimization_constraints = optimizer_config_manager.parse_constraints(
constraint)
else:
project_config_manager = container.project_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
project_parameters = [
QCParameter(key=k, value=v)
for k, v in project_config.get("parameters", {}).items()
]
if len(project_parameters) == 0:
raise MoreInfoError(
"The given project has no parameters to optimize",
"https://www.lean.io/docs/lean-cli/optimization/parameters")
optimization_strategy = optimizer_config_manager.configure_strategy(
cloud=False)
optimization_target = optimizer_config_manager.configure_target()
optimization_parameters = optimizer_config_manager.configure_parameters(
project_parameters, cloud=False)
optimization_constraints = optimizer_config_manager.configure_constraints(
)
if config is None:
# noinspection PyUnboundLocalVariable
config = {
"optimization-strategy":
optimization_strategy,
"optimization-strategy-settings": {
"$type":
"QuantConnect.Optimizer.Strategies.StepBaseOptimizationStrategySettings, QuantConnect.Optimizer",
"default-segment-amount": 10
},
"optimization-criterion": {
"target": optimization_target.target,
"extremum": optimization_target.extremum.value
},
"parameters":
[parameter.dict() for parameter in optimization_parameters],
"constraints": [
constraint.dict(by_alias=True)
for constraint in optimization_constraints
]
}
config["optimizer-close-automatically"] = True
config["results-destination-folder"] = "/Results"
config_path = output / "optimizer-config.json"
config_path.parent.mkdir(parents=True, exist_ok=True)
with config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps(config, indent=4) + "\n")
project_config_manager = container.project_config_manager()
cli_config_manager = container.cli_config_manager()
project_config = project_config_manager.get_project_config(
algorithm_file.parent)
engine_image = cli_config_manager.get_engine_image(
image or project_config.get("engine-image", None))
lean_config_manager = container.lean_config_manager()
lean_config = lean_config_manager.get_complete_lean_config(
"backtesting", algorithm_file, None)
if not output.exists():
output.mkdir(parents=True)
output_config_manager = container.output_config_manager()
lean_config["algorithm-id"] = str(
output_config_manager.get_optimization_id(output))
lean_config["messaging-handler"] = "QuantConnect.Messaging.Messaging"
lean_runner = container.lean_runner()
run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file, output,
None, release, detach)
run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
run_options["commands"].append(
"dotnet QuantConnect.Optimizer.Launcher.dll")
run_options["mounts"].append(
Mount(target="/Lean/Optimizer.Launcher/bin/Debug/config.json",
source=str(config_path),
type="bind",
read_only=True))
container.update_manager().pull_docker_image_if_necessary(
engine_image, update)
project_manager.copy_code(algorithm_file.parent, output / "code")
success = container.docker_manager().run_image(engine_image, **run_options)
logger = container.logger()
cli_root_dir = container.lean_config_manager().get_cli_root_directory()
relative_project_dir = project.relative_to(cli_root_dir)
relative_output_dir = output.relative_to(cli_root_dir)
if detach:
temp_manager = container.temp_manager()
temp_manager.delete_temporary_directories_when_done = False
logger.info(
f"Successfully started optimization for '{relative_project_dir}' in the '{run_options['name']}' container"
)
logger.info(f"The output will be stored in '{relative_output_dir}'")
logger.info(
"You can use Docker's own commands to manage the detached container"
)
elif success:
optimizer_logs = (output / "log.txt").read_text(encoding="utf-8")
groups = re.findall(r"ParameterSet: \(([^)]+)\) backtestId '([^']+)'",
optimizer_logs)
if len(groups) > 0:
optimal_parameters, optimal_id = groups[0]
optimal_results = json.loads(
(output / optimal_id /
f"{optimal_id}.json").read_text(encoding="utf-8"))
optimal_backtest = QCBacktest(
backtestId=optimal_id,
projectId=1,
status="",
name=optimal_id,
created=datetime.now(),
completed=True,
progress=1.0,
runtimeStatistics=optimal_results["RuntimeStatistics"],
statistics=optimal_results["Statistics"])
logger.info(
f"Optimal parameters: {optimal_parameters.replace(':', ': ').replace(',', ', ')}"
)
logger.info(f"Optimal backtest results:")
logger.info(optimal_backtest.get_statistics_table())
logger.info(
f"Successfully optimized '{relative_project_dir}' and stored the output in '{relative_output_dir}'"
)
else:
raise RuntimeError(
f"Something went wrong while running the optimization, the output is stored in '{relative_output_dir}'"
)