def _get_aztk_environment(cluster_id, worker_on_master, mixed_mode): envs = [] envs.append( batch_models.EnvironmentSetting(name="AZTK_MIXED_MODE", value=helpers.bool_env(mixed_mode))) envs.append( batch_models.EnvironmentSetting( name="AZTK_WORKER_ON_MASTER", value=helpers.bool_env(worker_on_master))) envs.append( batch_models.EnvironmentSetting(name="AZTK_CLUSTER_ID", value=cluster_id)) return envs
def generate_cluster_start_task( core_base_operations, zip_resource_file: batch_models.ResourceFile, cluster_id: str, gpu_enabled: bool, docker_repo: str = None, docker_run_options: str = None, file_shares: List[models.FileShare] = None, mixed_mode: bool = False, worker_on_master: bool = True, ): """ This will return the start task object for the pool to be created. :param cluster_id str: Id of the cluster(Used for uploading the resource files) :param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node """ resource_files = [zip_resource_file] spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT spark_container_name = constants.DOCKER_SPARK_CONTAINER_NAME spark_submit_logs_file = constants.SPARK_SUBMIT_LOGS_FILE # TODO use certificate environment_settings = ( __get_secrets_env(core_base_operations) + [ batch_models.EnvironmentSetting(name="SPARK_WEB_UI_PORT", value=spark_web_ui_port), batch_models.EnvironmentSetting(name="SPARK_WORKER_UI_PORT", value=spark_worker_ui_port), batch_models.EnvironmentSetting(name="SPARK_JOB_UI_PORT", value=spark_job_ui_port), batch_models.EnvironmentSetting(name="SPARK_CONTAINER_NAME", value=spark_container_name), batch_models.EnvironmentSetting(name="SPARK_SUBMIT_LOGS_FILE", value=spark_submit_logs_file), batch_models.EnvironmentSetting( name="AZTK_GPU_ENABLED", value=helpers.bool_env(gpu_enabled)), ] + __get_docker_credentials(core_base_operations) + _get_aztk_environment(cluster_id, worker_on_master, mixed_mode)) # start task command command = __cluster_install_cmd(zip_resource_file, gpu_enabled, docker_repo, docker_run_options, file_shares) return batch_models.StartTask( command_line=helpers.wrap_commands_in_shell(command), resource_files=resource_files, environment_settings=environment_settings, user_identity=POOL_ADMIN_USER_IDENTITY, wait_for_success=True, max_task_retry_count=2, )
def test_bool_env(): assert helpers.bool_env(True) == "true" assert helpers.bool_env(False) == "false" assert helpers.bool_env(None) == "false" assert helpers.bool_env("some") == "false"