def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]:
"""Retrieves the available datasets.
:param organization: the organization that will be charged
:return: the datasets which data can be downloaded from
"""
cloud_datasets = container.api_client().market.list_datasets()
data_information = _get_data_information(organization)
available_datasets = []
for cloud_dataset in cloud_datasets:
if cloud_dataset.delivery == QCDatasetDelivery.CloudOnly:
continue
datasource = data_information.datasources.get(str(cloud_dataset.id), None)
if datasource is None or isinstance(datasource, list):
if cloud_dataset.name != "Template Data Source Product":
name = cloud_dataset.name.strip()
vendor = cloud_dataset.vendorName.strip()
container.logger().debug(
f"There is no datasources entry for {name} by {vendor} (id {cloud_dataset.id})")
continue
available_datasets.append(Dataset(name=cloud_dataset.name.strip(),
vendor=cloud_dataset.vendorName.strip(),
categories=[tag.name.strip() for tag in cloud_dataset.tags],
options=datasource["options"],
paths=datasource["paths"],
requires_security_master=datasource["requiresSecurityMaster"]))
return available_datasets
def _start_iqconnect_if_necessary(lean_config: Dict[str, Any],
environment_name: str) -> None:
"""Starts IQConnect if the given environment uses IQFeed as data queue handler.
:param lean_config: the LEAN configuration that should be used
:param environment_name: the name of the environment
"""
environment = lean_config["environments"][environment_name]
if environment[
"data-queue-handler"] != "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler":
return
args = [
lean_config["iqfeed-iqconnect"], "-product",
lean_config["iqfeed-productName"], "-version",
lean_config["iqfeed-version"]
]
username = lean_config.get("iqfeed-username", "")
if username != "":
args.extend(["-login", username])
password = lean_config.get("iqfeed-password", "")
if password != "":
args.extend(["-password", password])
subprocess.Popen(args)
container.logger().info("Waiting 10 seconds for IQFeed to start")
time.sleep(10)
def _select_products_non_interactive(organization: QCFullOrganization,
datasets: List[Dataset],
ctx: click.Context) -> List[Product]:
"""Asks the user for the products that should be purchased and downloaded.
:param organization: the organization that will be charged
:param datasets: the available datasets
:param ctx: the click context of the invocation
:return: the list of products selected by the user
"""
dataset = next((d for d in datasets if d.name.lower() == ctx.params["dataset"].lower()), None)
if dataset is None:
raise RuntimeError(f"There is no dataset named '{ctx.params['dataset']}'")
if dataset.requires_security_master and not organization.has_security_master_subscription():
raise RuntimeError("\n".join([
f"Your organization needs to have an active Security Master subscription to download data from the '{dataset.name}' dataset",
f"You can add the subscription at https://www.quantconnect.com/datasets/quantconnect-security-master/pricing"
]))
option_results = OrderedDict()
invalid_options = []
missing_options = []
for option in dataset.options:
if option.condition is not None and not option.condition.check(option_results):
continue
user_input = ctx.params.get(option.id, None)
if user_input is None:
missing_options.append(f"--{option.id} <{option.get_placeholder()}>: {option.description}")
else:
try:
option_results[option.id] = option.configure_non_interactive(user_input)
except ValueError as error:
invalid_options.append(f"--{option.id}: {error}")
if len(invalid_options) > 0 or len(missing_options) > 0:
blocks = []
for label, lines in [["Invalid option", invalid_options], ["Missing option", missing_options]]:
if len(lines) > 0:
joined_lines = "\n".join(lines)
blocks.append(f"{label}{'s' if len(lines) > 1 else ''}:\n{joined_lines}")
raise RuntimeError("\n\n".join(blocks))
products = [Product(dataset=dataset, option_results=option_results)]
container.logger().info("Data that will be purchased and downloaded:")
_display_products(organization, products)
return products
def whoami() -> None:
"""Display who is logged in."""
logger = container.logger()
api_client = container.api_client()
cli_config_manager = container.cli_config_manager()
if cli_config_manager.user_id.get_value(
) is not None and cli_config_manager.api_token.get_value() is not None:
try:
organizations = api_client.organizations.get_all()
logged_in = True
except AuthenticationError:
logged_in = False
else:
logged_in = False
if not logged_in:
logger.info("You are not logged in")
return
personal_organization_id = next(o.id for o in organizations
if o.ownerName == "You")
personal_organization = api_client.organizations.get(
personal_organization_id)
member = next(m for m in personal_organization.members if m.isAdmin)
logger.info(f"You are logged in as {member.name} ({member.email})")
def _compile_lean(lean_dir: Path) -> None:
"""Compiles LEAN's C# code.
:param lean_dir: the directory containing the LEAN repository
"""
logger = container.logger()
logger.info(f"Compiling the C# code in '{lean_dir}'")
docker_manager = container.docker_manager()
docker_manager.create_volume("lean_cli_nuget")
success = docker_manager.run_image(
CUSTOM_FOUNDATION_IMAGE,
entrypoint=["dotnet", "build", f"/LeanCLI"],
environment={
"DOTNET_CLI_TELEMETRY_OPTOUT": "true",
"DOTNET_NOLOGO": "true"
},
volumes={
str(lean_dir): {
"bind": "/LeanCLI",
"mode": "rw"
},
"lean_cli_nuget": {
"bind": "/root/.nuget/packages",
"mode": "rw"
}
})
if not success:
raise RuntimeError(
"Something went wrong while running dotnet build, see the logs above for more information"
)
def _log_notification_methods(methods: List[QCNotificationMethod]) -> None:
"""Logs a list of notification methods."""
logger = container.logger()
email_methods = [
method for method in methods
if isinstance(method, QCEmailNotificationMethod)
]
email_methods = "None" if len(email_methods) == 0 else ", ".join(
method.address for method in email_methods)
webhook_methods = [
method for method in methods
if isinstance(method, QCWebhookNotificationMethod)
]
webhook_methods = "None" if len(webhook_methods) == 0 else ", ".join(
method.address for method in webhook_methods)
sms_methods = [
method for method in methods
if isinstance(method, QCSMSNotificationMethod)
]
sms_methods = "None" if len(sms_methods) == 0 else ", ".join(
method.phoneNumber for method in sms_methods)
logger.info(f"Email notifications: {email_methods}")
logger.info(f"Webhook notifications: {webhook_methods}")
logger.info(f"SMS notifications: {sms_methods}")
def _verify_accept_agreement(organization: QCFullOrganization, open_browser: bool) -> None:
""" Verifies that the user has accepted the agreement.
If they haven't, asks the user to accept the CLI API Access and Data Agreement.
If they have, reminds them of the agreement and moves on.
The API will enforce signing the agreement at the end of the day but this is how we keep it in the process of the CLI
:param organization: the organization that the user selected
:param open_browser: whether the CLI should automatically open the agreement in the browser
"""
logger = container.logger()
api_client = container.api_client()
info = api_client.data.get_info(organization.id)
# Is signed
if organization.data.current:
logger.info(_presigned_terms.format(link=info.agreement, signed_time=datetime.fromtimestamp(organization.data.signedTime)))
sleep(1)
else:
if open_browser:
webbrowser.open(info.agreement)
logger.info(f"Go to the following url to accept the CLI API Access and Data Agreement:")
logger.info(info.agreement)
logger.info("Waiting until the CLI API Access and Data Agreement has been accepted...")
container.task_manager().poll(
make_request=lambda: api_client.organizations.get(organization.id),
is_done=lambda data: data.data.current != False
)
def _build_image(dockerfile: Path, base_image: Optional[DockerImage],
target_image: DockerImage) -> None:
"""Builds a Docker image.
:param dockerfile: the path to the Dockerfile to build
:param base_image: the base image to use, or None if the default should be used
:param target_image: the name of the new image
"""
logger = container.logger()
if base_image is not None:
logger.info(
f"Building '{target_image}' from '{dockerfile}' using '{base_image}' as base image"
)
else:
logger.info(f"Building '{target_image}' from '{dockerfile}'")
if not dockerfile.is_file():
raise RuntimeError(f"'{dockerfile}' does not exist")
current_content = dockerfile.read_text(encoding="utf-8")
if base_image is not None:
new_content = re.sub(r"^FROM.*$",
f"FROM {base_image}",
current_content,
flags=re.MULTILINE)
dockerfile.write_text(new_content, encoding="utf-8")
try:
docker_manager = container.docker_manager()
docker_manager.build_image(dockerfile, target_image)
finally:
if base_image is not None:
dockerfile.write_text(current_content, encoding="utf-8")
def _display_estimate(cloud_project: QCProject,
finished_compile: QCCompileWithLogs,
organization: QCFullOrganization, name: str,
strategy: str, target: OptimizationTarget,
parameters: List[OptimizationParameter],
constraints: List[OptimizationConstraint],
node: NodeType, parallel_nodes: int) -> None:
"""Displays the estimated optimization time and cost."""
api_client = container.api_client()
estimate = api_client.optimizations.estimate(cloud_project.projectId,
finished_compile.compileId,
name, strategy, target,
parameters, constraints,
node.name, parallel_nodes)
backtest_count = _calculate_backtest_count(parameters)
hours = _calculate_hours(estimate.time, backtest_count)
batch_time = ceil((hours * 100) / parallel_nodes) / 100
batch_cost = max(0.01, ceil(node.price * hours * 100) / 100)
logger = container.logger()
logger.info(f"Estimated number of backtests: {backtest_count:,}")
logger.info(f"Estimated batch time: {_format_hours(batch_time)}")
logger.info(f"Estimated batch cost: ${batch_cost:,.2f}")
logger.info(
f"Organization balance: {organization.credit.balance:,.0f} QCC (${organization.credit.balance / 100:,.2f})"
)
def _remove_python(project_dir: Path, name: str) -> None:
"""Removes a custom Python library from a Python project.
Removes the library from the project's requirements.txt file.
:param project_dir: the path to the project directory
:param name: the name of the library to remove
"""
logger = container.logger()
path_manager = container.path_manager()
requirements_file = project_dir / "requirements.txt"
logger.info(
f"Removing {name} from '{path_manager.get_relative_path(requirements_file)}'"
)
if not requirements_file.is_file():
return
requirements_content = requirements_file.read_text(encoding="utf-8")
new_lines = []
for line in requirements_content.splitlines():
try:
requirement = Requirement.parse(line)
if requirement.name.lower() != name.lower():
new_lines.append(line)
except ValueError:
new_lines.append(line)
new_content = "\n".join(new_lines).strip()
new_content = new_content + "\n" if len(new_content) > 0 else new_content
requirements_file.write_text(new_content, encoding="utf-8")
def login(user_id: Optional[str], api_token: Optional[str]) -> None:
"""Log in with a QuantConnect account.
If user id or API token is not provided an interactive prompt will show.
Credentials are stored in ~/.lean/credentials and are removed upon running `lean logout`.
"""
logger = container.logger()
credentials_storage = container.credentials_storage()
if user_id is None or api_token is None:
logger.info(
"Your user id and API token are needed to make authenticated requests to the QuantConnect API"
)
logger.info(
"You can request these credentials on https://www.quantconnect.com/account"
)
logger.info(f"Both will be saved in {credentials_storage.file}")
if user_id is None:
user_id = click.prompt("User id")
if api_token is None:
api_token = click.prompt("API token")
api_client = container.api_client(user_id=user_id, api_token=api_token)
if not api_client.is_authenticated():
raise RuntimeError("Credentials are invalid")
cli_config_manager = container.cli_config_manager()
cli_config_manager.user_id.set_value(user_id)
cli_config_manager.api_token.set_value(api_token)
logger.info("Successfully logged in")
def create_project(name: str, language: str) -> None:
"""Create a new project containing starter code.
If NAME is a path containing subdirectories those will be created automatically.
The default language can be set using `lean config set default-language python/csharp`.
"""
cli_config_manager = container.cli_config_manager()
language = language if language is not None else cli_config_manager.default_language.get_value(
)
if language is None:
raise MoreInfoError(
"Please specify a language with --language or set the default language using `lean config set default-language python/csharp`",
"https://www.quantconnect.com/docs/v2/lean-cli/tutorials/project-management"
)
full_path = Path.cwd() / name
path_validator = container.path_validator()
if not path_validator.is_path_valid(full_path):
raise MoreInfoError(
f"'{name}' is not a valid path",
"https://www.quantconnect.com/docs/v2/lean-cli/user-guides/troubleshooting#02-Common-errors"
)
if full_path.exists():
raise RuntimeError(
f"A project named '{name}' already exists, please choose a different name"
)
else:
project_manager = container.project_manager()
project_manager.create_new_project(
full_path,
QCLanguage.Python if language == "python" else QCLanguage.CSharp)
# Convert the project name into a valid class name by removing all non-alphanumeric characters
class_name = re.sub(f"[^a-zA-Z0-9]", "",
"".join(map(_capitalize, name.split(" "))))
if language == "python":
with (full_path / "main.py").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_MAIN.replace("$NAME$", class_name))
else:
with (full_path / "Main.cs").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_CSHARP_MAIN.replace("$NAME$", class_name))
with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_NOTEBOOK if language ==
"python" else DEFAULT_CSHARP_NOTEBOOK)
with (full_path / "config.json").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_CONFIG if language ==
"python" else DEFAULT_CSHARP_CONFIG)
logger = container.logger()
logger.info(
f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'"
)
def build(root: Path, tag: str) -> None:
"""Build Docker images of your own version of LEAN and the Alpha Streams SDK.
\b
ROOT must point to a directory containing the LEAN repository and the Alpha Streams SDK repository:
https://github.com/QuantConnect/Lean & https://github.com/QuantConnect/AlphaStreams
When ROOT is not given, the current directory is used as root directory.
\b
This command performs the following actions:
1. The lean-cli/foundation:latest image is built from Lean/DockerfileLeanFoundation(ARM).
2. LEAN is compiled in a Docker container using the lean-cli/foundation:latest image.
3. The Alpha Streams SDK is compiled in a Docker container using the lean-cli/foundation:latest image.
4. The lean-cli/engine:latest image is built from Lean/Dockerfile using lean-cli/foundation:latest as base image.
5. The lean-cli/research:latest image is built from Lean/DockerfileJupyter using lean-cli/engine:latest as base image.
6. The default engine image is set to lean-cli/engine:latest.
7. The default research image is set to lean-cli/research:latest.
"""
lean_dir = root / "Lean"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/Lean to '{lean_dir}'"
)
alpha_streams_dir = root / "AlphaStreams"
if not lean_dir.is_dir():
raise RuntimeError(
f"Please clone https://github.com/QuantConnect/AlphaStreams to '{alpha_streams_dir}'"
)
(root / "DataLibraries").mkdir(exist_ok=True)
if platform.machine() in ["arm64", "aarch64"]:
foundation_dockerfile = lean_dir / "DockerfileLeanFoundationARM"
else:
foundation_dockerfile = lean_dir / "DockerfileLeanFoundation"
custom_foundation_image = DockerImage(name="lean-cli/foundation", tag=tag)
custom_engine_image = DockerImage(name="lean-cli/engine", tag=tag)
custom_research_image = DockerImage(name="lean-cli/research", tag=tag)
_build_image(root, foundation_dockerfile, None, custom_foundation_image)
_compile_csharp(root, lean_dir, custom_foundation_image)
_compile_csharp(root, alpha_streams_dir, custom_foundation_image)
_build_image(root, lean_dir / "Dockerfile", custom_foundation_image,
custom_engine_image)
_build_image(root, lean_dir / "DockerfileJupyter", custom_engine_image,
custom_research_image)
logger = container.logger()
cli_config_manager = container.cli_config_manager()
logger.info(f"Setting default engine image to '{custom_engine_image}'")
cli_config_manager.engine_image.set_value(str(custom_engine_image))
logger.info(f"Setting default research image to '{custom_research_image}'")
cli_config_manager.research_image.set_value(str(custom_research_image))
def create_project(name: str, language: str) -> None:
"""Create a new project containing starter code.
If NAME is a path containing subdirectories those will be created automatically.
The default language can be set using `lean config set default-language python/csharp`.
"""
cli_config_manager = container.cli_config_manager()
language = language if language is not None else cli_config_manager.default_language.get_value()
if language is None:
raise MoreInfoError(
"Please specify a language with --language or set the default language using `lean config set default-language python/csharp`",
"https://www.lean.io/docs/lean-cli/projects/project-management")
full_path = Path.cwd() / name
if not container.path_manager().is_path_valid(full_path):
raise MoreInfoError(f"'{name}' is not a valid path",
"https://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors")
is_library_project = False
try:
library_dir = container.lean_config_manager().get_cli_root_directory() / "Library"
is_library_project = library_dir in full_path.parents
except:
# get_cli_root_directory() raises an error if there is no such directory
pass
if is_library_project and language == "python" and not full_path.name.isidentifier():
raise RuntimeError(
f"'{full_path.name}' is not a valid Python identifier, which is required for Python library projects to be importable")
if full_path.exists():
raise RuntimeError(f"A project named '{name}' already exists, please choose a different name")
else:
project_manager = container.project_manager()
project_manager.create_new_project(full_path, QCLanguage.Python if language == "python" else QCLanguage.CSharp)
# Convert the project name into a valid class name by removing all non-alphanumeric characters
class_name = re.sub(f"[^a-zA-Z0-9]", "", "".join(map(_capitalize, full_path.name.split(" "))))
if language == "python":
main_name = "main.py"
main_content = DEFAULT_PYTHON_MAIN if not is_library_project else LIBRARY_PYTHON_MAIN
else:
main_name = "Main.cs"
main_content = DEFAULT_CSHARP_MAIN if not is_library_project else LIBRARY_CSHARP_MAIN
with (full_path / main_name).open("w+", encoding="utf-8") as file:
file.write(main_content.replace("$CLASS_NAME$", class_name).replace("$PROJECT_NAME$", full_path.name))
with (full_path / "research.ipynb").open("w+", encoding="utf-8") as file:
file.write(DEFAULT_PYTHON_NOTEBOOK if language == "python" else DEFAULT_CSHARP_NOTEBOOK)
logger = container.logger()
logger.info(f"Successfully created {'Python' if language == 'python' else 'C#'} project '{name}'")
def stop() -> None:
"""Stop the local GUI."""
docker_manager = container.docker_manager()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
raise RuntimeError(
"The local GUI is not running, you can start it using `lean gui start`"
)
gui_container.stop()
for container_name in docker_manager.get_running_containers():
if container_name.startswith("lean_cli_gui_research_"):
docker_manager.get_container_by_name(container_name).stop()
container.logger().info(f"Successfully stopped the local GUI")
def _select_products(organization: QCFullOrganization) -> List[Product]:
"""Asks the user for the products that should be purchased and downloaded.
:return: the list of products selected by the user
"""
products = []
logger = container.logger()
security_product_classes = [
CFDProduct, CryptoProduct, EquityProduct, EquityOptionProduct,
ForexProduct, FutureProduct
]
alternative_product_classes = [
CBOEProduct, FREDProduct, SECProduct, USTreasuryProduct,
USEnergyProduct
]
while True:
initial_type = logger.prompt_list(
"Select whether you want to download security data or alternative data",
[
Option(id="security", label="Security data"),
Option(id="alternative", label="Alternative data")
])
if initial_type == "security":
product_classes = security_product_classes
product_name_question = "Select the security type"
else:
product_classes = alternative_product_classes
product_name_question = "Select the data type"
product_class = logger.prompt_list(product_name_question, [
Option(id=c, label=c.get_product_name()) for c in product_classes
])
new_products = product_class.build(organization)
current_files = [
data_file.file
for data_file in _get_data_files(organization, products)
]
for new_product in new_products:
new_files = new_product.get_data_files()
if len(set(new_files) - set(current_files)) > 0:
products.append(new_product)
logger.info("Selected data:")
_display_products(organization, products)
if not click.confirm("Do you want to download more data?"):
break
return products
def status(project: str) -> None:
"""Show the live trading status of a project in the cloud.
PROJECT must be the name or the id of the project to show the status for.
"""
logger = container.logger()
api_client = container.api_client()
cloud_project_manager = container.cloud_project_manager()
cloud_project = cloud_project_manager.get_cloud_project(project, False)
live_algorithm = next((d for d in api_client.live.get_all()
if d.projectId == cloud_project.projectId), None)
logger.info(f"Project id: {cloud_project.projectId}")
logger.info(f"Project name: {cloud_project.name}")
logger.info(f"Project url: {cloud_project.get_url()}")
if live_algorithm is None:
logger.info("Live status: Not deployed")
return
live_status = {
QCLiveAlgorithmStatus.DeployError: "Deploy error",
QCLiveAlgorithmStatus.InQueue: "In queue",
QCLiveAlgorithmStatus.RuntimeError: "Runtime error",
QCLiveAlgorithmStatus.LoggingIn: "Logging in"
}.get(live_algorithm.status, live_algorithm.status.value)
brokerage_name = next(
(b.name
for b in all_cloud_brokerages if b.id == live_algorithm.brokerage),
live_algorithm.brokerage)
if brokerage_name == "PaperBrokerage":
brokerage_name = "Paper Trading"
logger.info(f"Live status: {live_status}")
logger.info(f"Live id: {live_algorithm.deployId}")
logger.info(f"Live url: {live_algorithm.get_url()}")
logger.info(f"Brokerage: {brokerage_name}")
logger.info(
f"Launched: {live_algorithm.launched.strftime('%Y-%m-%d %H:%M:%S')} UTC"
)
if live_algorithm.stopped is not None:
logger.info(
f"Stopped: {live_algorithm.stopped.strftime('%Y-%m-%d %H:%M:%S')} UTC"
)
if live_algorithm.error != "":
logger.info("Error:")
logger.info(live_algorithm.error)
def unset(key: str) -> None:
"""Unset a configurable option.
Run `lean config list` to show all available options.
"""
cli_config_manager = container.cli_config_manager()
option = cli_config_manager.get_option_by_key(key)
option.unset()
logger = container.logger()
logger.info(f"Successfully unset '{key}'")
def _select_products_interactive(organization: QCFullOrganization, datasets: List[Dataset]) -> List[Product]:
"""Asks the user for the products that should be purchased and downloaded.
:param organization: the organization that will be charged
:param datasets: the available datasets
:return: the list of products selected by the user
"""
products = []
logger = container.logger()
category_options = {}
for dataset in datasets:
for category in dataset.categories:
if category in category_options:
continue
dataset_count = len(list(dataset for dataset in datasets if category in dataset.categories))
category_options[category] = Option(
id=category,
label=f"{category} ({dataset_count} dataset{'s' if dataset_count > 1 else ''})"
)
category_options = sorted(category_options.values(), key=lambda opt: opt.label)
while True:
category = logger.prompt_list("Select a category", category_options)
available_datasets = sorted((d for d in datasets if category in d.categories), key=lambda d: d.name)
dataset: Dataset = logger.prompt_list("Select a dataset",
[Option(id=d, label=d.name) for d in available_datasets])
if dataset.requires_security_master and not organization.has_security_master_subscription():
logger.warn("\n".join([
f"Your organization needs to have an active Security Master subscription to download data from the '{dataset.name}' dataset",
f"You can add the subscription at https://www.quantconnect.com/datasets/quantconnect-security-master/pricing"
]))
continue
option_results = OrderedDict()
for option in dataset.options:
if option.condition is None or option.condition.check(option_results):
option_results[option.id] = option.configure_interactive()
products.append(Product(dataset=dataset, option_results=option_results))
logger.info("Selected data:")
_display_products(organization, products)
if not click.confirm("Do you want to download more data?"):
break
return products
def _display_products(organization: QCFullOrganization, products: List[Product]) -> None:
"""Previews a list of products in pretty tables.
:param organization: the organization the user selected
:param products: the products to display
"""
logger = container.logger()
table = Table(box=box.SQUARE)
for column in ["Dataset", "Vendor", "Details", "File count", "Price"]:
table.add_column(column, overflow="fold")
summed_price = 0
for product in products:
details = []
for option_id, result in product.option_results.items():
option = next(o for o in product.dataset.options if o.id == option_id)
if result is not None:
label = option.label
if isinstance(result.value, list):
if len(result.value) > 1:
label = label.replace("(s)", "s")
else:
label = label.replace("(s)", "")
details.append(f"{label}: {result.label}")
if len(details) == 0:
details.append("-")
mapped_files = _map_data_files_to_vendors(organization, product.get_data_files())
price = sum(data_file.vendor.price for data_file in mapped_files)
summed_price += price
table.add_row(product.dataset.name,
product.dataset.vendor,
"\n".join(details),
f"{len(mapped_files):,.0f}",
f"{price:,.0f} QCC")
logger.info(table)
all_data_files = _get_data_files(organization, products)
total_price = sum(data_file.vendor.price for data_file in all_data_files)
if total_price != summed_price:
logger.warn("The total price is less than the sum of all separate prices because there is overlapping data")
logger.info(f"Total price: {total_price:,.0f} QCC")
logger.info(f"Organization balance: {organization.credit.balance:,.0f} QCC")
def invoke(self, ctx):
if self._requires_lean_config:
try:
# This method will throw if the directory cannot be found
container.lean_config_manager().get_cli_root_directory()
except Exception:
# Abort with a display-friendly error message if the command requires a Lean config
raise MoreInfoError(
"This command requires a Lean configuration file, run `lean init` in an empty directory to create one, or specify the file to use with --lean-config",
"https://www.lean.io/docs/lean-cli/user-guides/troubleshooting#02-Common-errors")
if self._requires_docker and "pytest" not in sys.modules:
# The CLI uses temporary directories in /tmp because sometimes it may leave behind files owned by root
# These files cannot be deleted by the CLI itself, so we rely on the OS to empty /tmp on reboot
# The Snap version of Docker does not provide access to files outside $HOME, so we can't support it
if platform.system() == "Linux":
docker_path = shutil.which("docker")
if docker_path is not None and docker_path.startswith("/snap"):
raise MoreInfoError(
"The Lean CLI does not work with the Snap version of Docker, please re-install Docker via the official installation instructions",
"https://docs.docker.com/engine/install/")
# A usual Docker installation on Linux requires the user to use sudo to run Docker
# If we detect that this is the case and the CLI was started without sudo we elevate automatically
if platform.system() == "Linux" and os.getuid() != 0 and container.docker_manager().is_missing_permission():
container.logger().info(
"This command requires access to Docker, you may be asked to enter your password")
args = ["sudo", "--preserve-env=HOME", sys.executable, *sys.argv]
os.execlp(args[0], *args)
update_manager = container.update_manager()
update_manager.show_announcements()
result = super().invoke(ctx)
update_manager.warn_if_cli_outdated()
return result
def set(key: str, value: str) -> None:
"""Set a configurable option.
Run `lean config list` to show all available options.
"""
cli_config_manager = container.cli_config_manager()
option = cli_config_manager.get_option_by_key(key)
option.set_value(value)
logger = container.logger()
logger.info(
f"Successfully updated the value of '{key}' to '{option.get_value()}'")
def main() -> None:
"""This function is the entrypoint when running a Lean command in a terminal."""
try:
lean.main()
except Exception as exception:
logger = container.logger()
if logger.debug_logging_enabled:
logger.debug(traceback.format_exc().strip())
else:
logger.error(f"Error: {exception}")
sys.exit(1)
def restart(no_open: bool) -> None:
"""Restart the local GUI and open it in the browser."""
logger = container.logger()
docker_manager = container.docker_manager()
gui_container = docker_manager.get_container_by_name(
LOCAL_GUI_CONTAINER_NAME)
if gui_container is None or gui_container.status != "running":
raise RuntimeError(
"The local GUI is not running, you can start it using `lean gui start`"
)
logger.info("Restarting the local GUI's Docker container")
gui_container.restart()
port = gui_container.ports["5612/tcp"][0]["HostPort"]
url = f"http://localhost:{port}/"
# Wait until the GUI is running again
while True:
if LOCAL_GUI_CONTAINER_NAME not in docker_manager.get_running_containers(
):
docker_manager.show_logs(LOCAL_GUI_CONTAINER_NAME)
raise RuntimeError(
"Something went wrong while restarting the local GUI, see the logs above for more information"
)
try:
requests.get(url)
break
except requests.exceptions.ConnectionError:
time.sleep(0.25)
container.logger().info(
f"The local GUI has restarted and is running on {url}")
if not no_open:
webbrowser.open(url)
def _select_organization() -> QCFullOrganization:
"""Asks the user for the organization that should be used.
:return: the selected organization
"""
api_client = container.api_client()
organizations = api_client.organizations.get_all()
options = [Option(id=organization.id, label=organization.name) for organization in organizations]
logger = container.logger()
organization_id = logger.prompt_list("Select the organization to purchase and download data with", options)
return api_client.organizations.get(organization_id)
def build(cls, lean_config: Dict[str, Any],
logger: Logger) -> LeanConfigConfigurer:
api_client = container.api_client()
organizations = api_client.organizations.get_all()
options = [
Option(id=organization.id, label=organization.name)
for organization in organizations
]
logger = container.logger()
organization_id = logger.prompt_list(
"Select the organization to purchase and download data with",
options)
return QuantConnectDataProvider(organization_id)
def _select_organization() -> QCMinimalOrganization:
"""Asks the user for the organization that should be charged when downloading data.
:return: the selected organization
"""
api_client = container.api_client()
organizations = api_client.organizations.get_all()
options = [
Option(id=organization, label=organization.name)
for organization in organizations
]
logger = container.logger()
return logger.prompt_list(
"Select the organization to purchase and download data with", options)
def _add_csharp(project_dir: Path, name: str, version: Optional[str],
no_local: bool) -> None:
"""Adds a custom C# library to a C# project.
Adds the library to the project's .csproj file, and restores the project is dotnet is on the user's PATH.
:param project_dir: the path to the project directory
:param name: the name of the library to add
:param version: the version of the library to use, or None to pin to the latest version
:param no_local: whether restoring the packages locally must be skipped
"""
logger = container.logger()
path_manager = container.path_manager()
if version is None:
logger.info("Retrieving latest available version from NuGet")
name, version = _get_nuget_package(name)
csproj_file = next(p for p in project_dir.iterdir()
if p.name.endswith(".csproj"))
logger.info(
f"Adding {name} {version} to '{path_manager.get_relative_path(csproj_file)}'"
)
original_csproj_content = csproj_file.read_text(encoding="utf-8")
_add_csharp_package_to_csproj(csproj_file, name, version)
if not no_local and shutil.which("dotnet") is not None:
logger.info(
f"Restoring packages in '{path_manager.get_relative_path(project_dir)}' to provide local autocomplete"
)
process = subprocess.run(
["dotnet", "restore", str(csproj_file)], cwd=project_dir)
if process.returncode != 0:
logger.warn(
f"Reverting the changes to '{path_manager.get_relative_path(csproj_file)}'"
)
csproj_file.write_text(original_csproj_content, encoding="utf-8")
raise RuntimeError(
"Something went wrong while restoring packages, see the logs above for more information"
)
def _migrate_dotnet_5_csharp_rider(project_dir: Path) -> None:
made_changes = False
for dir_name in [
f".idea.{project_dir.stem}", f".idea.{project_dir.stem}.dir"
]:
workspace_xml_path = project_dir / ".idea" / dir_name / ".idea" / "workspace.xml"
if not workspace_xml_path.is_file():
continue
current_content = ElementTree.fromstring(
workspace_xml_path.read_text(encoding="utf-8"))
run_manager = current_content.find(".//component[@name='RunManager']")
if run_manager is None:
continue
config = run_manager.find(
".//configuration[@name='Debug with Lean CLI']")
if config is None:
continue
run_manager.remove(config)
new_content = ElementTree.tostring(current_content,
encoding="utf-8",
method="xml").decode("utf-8")
workspace_xml_path.write_text(new_content, encoding="utf-8")
made_changes = True
if made_changes:
container.project_manager().generate_rider_config()
logger = container.logger()
logger.warn(
"Your run configuration has been updated to work with the .NET 5 version of LEAN"
)
logger.warn("Please restart Rider and start debugging again")
logger.warn(
"See https://www.quantconnect.com/docs/v2/lean-cli/tutorials/backtesting/debugging-local-backtests#05-C-and-Rider for the updated instructions"
)
raise click.Abort()
def main() -> None:
"""This function is the entrypoint when running a Lean command in a terminal."""
try:
lean.main(standalone_mode=False)
temp_manager = container.temp_manager()
if temp_manager.delete_temporary_directories_when_done:
temp_manager.delete_temporary_directories()
except Exception as exception:
logger = container.logger()
logger.debug(traceback.format_exc().strip())
if isinstance(exception, ValidationError) and hasattr(exception, "input_value"):
logger.debug("Value that failed validation:")
logger.debug(exception.input_value)
logger.error(f"Error: {exception}")
elif isinstance(exception, MoreInfoError):
logger.error(f"Error: {exception}")
logger.error(f"Visit {exception.link} for more information")
elif isinstance(exception, click.UsageError):
io = StringIO()
exception.show(file=io)
exception_str = io.getvalue().strip()
exception_str = exception_str.replace("Try 'lean", "\nTry 'lean")
exception_str = exception_str.replace("for help.",
"for help or go to the following url for a list of common errors:\nhttps://www.lean.io/docs/lean-cli/key-concepts/troubleshooting#02-Common-Errors")
container.update_manager().warn_if_cli_outdated(force=True)
logger.info(exception_str)
elif isinstance(exception, click.Abort):
logger.info("Aborted!")
elif isinstance(exception, requests.exceptions.ConnectionError):
logger.error(f"Error: {exception}")
logger.error("It looks like you don't have an internet connection, please check your system settings")
else:
logger.error(f"Error: {exception}")
temp_manager = container.temp_manager()
if temp_manager.delete_temporary_directories_when_done:
temp_manager.delete_temporary_directories()
sys.exit(1)
