def _get_aztk_environment(cluster_id, worker_on_master, mixed_mode):
envs = []
envs.append(
batch_models.EnvironmentSetting(name="AZTK_MIXED_MODE",
value=helpers.bool_env(mixed_mode)))
envs.append(
batch_models.EnvironmentSetting(
name="AZTK_WORKER_ON_MASTER",
value=helpers.bool_env(worker_on_master)))
envs.append(
batch_models.EnvironmentSetting(name="AZTK_CLUSTER_ID",
value=cluster_id))
return envs
def generate_cluster_start_task(
core_base_operations,
zip_resource_file: batch_models.ResourceFile,
cluster_id: str,
gpu_enabled: bool,
docker_repo: str = None,
docker_run_options: str = None,
file_shares: List[models.FileShare] = None,
mixed_mode: bool = False,
worker_on_master: bool = True,
):
"""
This will return the start task object for the pool to be created.
:param cluster_id str: Id of the cluster(Used for uploading the resource files)
:param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
"""
resource_files = [zip_resource_file]
spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT
spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT
spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT
spark_container_name = constants.DOCKER_SPARK_CONTAINER_NAME
spark_submit_logs_file = constants.SPARK_SUBMIT_LOGS_FILE
# TODO use certificate
environment_settings = (
__get_secrets_env(core_base_operations) + [
batch_models.EnvironmentSetting(name="SPARK_WEB_UI_PORT",
value=spark_web_ui_port),
batch_models.EnvironmentSetting(name="SPARK_WORKER_UI_PORT",
value=spark_worker_ui_port),
batch_models.EnvironmentSetting(name="SPARK_JOB_UI_PORT",
value=spark_job_ui_port),
batch_models.EnvironmentSetting(name="SPARK_CONTAINER_NAME",
value=spark_container_name),
batch_models.EnvironmentSetting(name="SPARK_SUBMIT_LOGS_FILE",
value=spark_submit_logs_file),
batch_models.EnvironmentSetting(
name="AZTK_GPU_ENABLED", value=helpers.bool_env(gpu_enabled)),
] + __get_docker_credentials(core_base_operations) +
_get_aztk_environment(cluster_id, worker_on_master, mixed_mode))
# start task command
command = __cluster_install_cmd(zip_resource_file, gpu_enabled,
docker_repo, docker_run_options,
file_shares)
return batch_models.StartTask(
command_line=helpers.wrap_commands_in_shell(command),
resource_files=resource_files,
environment_settings=environment_settings,
user_identity=POOL_ADMIN_USER_IDENTITY,
wait_for_success=True,
max_task_retry_count=2,
)
def test_bool_env():
assert helpers.bool_env(True) == "true"
assert helpers.bool_env(False) == "false"
assert helpers.bool_env(None) == "false"
assert helpers.bool_env("some") == "false"