def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster = spark_client.get_cluster(args.cluster_id) cluster_config = spark_client.get_cluster_config(args.cluster_id) ssh_conf = SshConfig() ssh_conf.merge( cluster_id=args.cluster_id, username=args.username, job_ui_port=args.jobui, job_history_ui_port=args.jobhistoryui, web_ui_port=args.webui, host=args.host, connect=args.connect, internal=args.internal) log.info("-------------------------------------------") utils.log_property("spark cluster id", ssh_conf.cluster_id) utils.log_property("open webui", "{0}{1}".format(http_prefix, ssh_conf.web_ui_port)) utils.log_property("open jobui", "{0}{1}".format(http_prefix, ssh_conf.job_ui_port)) utils.log_property("open jobhistoryui", "{0}{1}".format(http_prefix, ssh_conf.job_history_ui_port)) print_plugin_ports(cluster_config) utils.log_property("ssh username", ssh_conf.username) utils.log_property("connect", ssh_conf.connect) log.info("-------------------------------------------") try: shell_out_ssh(spark_client, ssh_conf) except OSError: # no ssh client is found, falling back to pure python native_python_ssh_into_master(spark_client, cluster, ssh_conf, args.password)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) log.info('-------------------------------------------') log.info('spark cluster id: {}'.format(args.cluster_id)) log.info('username: {}'.format(args.username)) log.info('-------------------------------------------') if args.ssh_key: ssh_key = args.ssh_key else: ssh_key = spark_client.secrets_config.ssh_pub_key ssh_key, password = utils.get_ssh_key_or_prompt( ssh_key, args.username, args.password, spark_client.secrets_config) spark_client.create_user(cluster_id=args.cluster_id, username=args.username, password=password, ssh_key=ssh_key) if password: log.info('password: %s', '*' * len(password)) elif ssh_key: log.info('ssh public key: %s', ssh_key) log.info('-------------------------------------------')
def get_spark_client(): # load secrets # note: this assumes secrets are set up in .aztk/secrets tenant_id = os.environ.get("TENANT_ID") client_id = os.environ.get("CLIENT_ID") credential = os.environ.get("CREDENTIAL") batch_account_resource_id = os.environ.get("BATCH_ACCOUNT_RESOURCE_ID") storage_account_resource_id = os.environ.get("STORAGE_ACCOUNT_RESOURCE_ID") ssh_pub_key = os.environ.get("ID_RSA_PUB") ssh_private_key = os.environ.get("ID_RSA") keys = [ tenant_id, client_id, credential, batch_account_resource_id, storage_account_resource_id, ssh_private_key, ssh_pub_key ] spark_client = None if all(keys): spark_client = aztk.spark.Client( aztk.spark.models.SecretsConfiguration( service_principal=aztk.spark.models. ServicePrincipalConfiguration( tenant_id=tenant_id, client_id=client_id, credential=credential, batch_account_resource_id=batch_account_resource_id, storage_account_resource_id=storage_account_resource_id), ssh_pub_key=ssh_pub_key, ssh_priv_key=ssh_private_key)) else: # fallback to local secrets if environment variables don't exist spark_client = aztk.spark.Client(config.load_aztk_secrets()) return spark_client
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) clusters = spark_client.list_clusters() if args.quiet: utils.print_clusters_quiet(clusters) else: utils.print_clusters(clusters)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_ids = args.cluster_ids for cluster_id in cluster_ids: if not args.force: if not args.keep_logs: log.warning( "All logs persisted for this cluster will be deleted.") confirmation_cluster_id = input( "Please confirm the id of the cluster you wish to delete [{}]: " .format(cluster_id)) if confirmation_cluster_id != cluster_id: log.error( "Confirmation cluster id does not match. Please try again." ) return if spark_client.cluster.delete(id=cluster_id, keep_logs=args.keep_logs): log.info("Deleting cluster %s", cluster_id) else: log.error( "Cluster with id '%s' doesn't exist or was already deleted.", cluster_id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) job_conf = JobConfig() job_conf.merge(args.job_id, args.job_conf) # by default, load spark configuration files in .aztk/ spark_configuration = config.load_aztk_spark_config() # overwrite with values in job_conf if they exist if job_conf.spark_defaults_conf: spark_configuration.spark_defaults_conf = job_conf.spark_defaults_conf if job_conf.spark_env_sh: spark_configuration.spark_env_sh = job_conf.spark_env_sh if job_conf.core_site_xml: spark_configuration.core_site_xml = job_conf.core_site_xml job_configuration = aztk.spark.models.JobConfiguration( id=job_conf.id, applications=job_conf.applications, custom_scripts=job_conf.custom_scripts, spark_configuration=spark_configuration, vm_size=job_conf.vm_size, docker_repo=job_conf.docker_repo, max_dedicated_nodes=job_conf.max_dedicated_nodes, max_low_pri_nodes=job_conf.max_low_pri_nodes, subnet_id=job_conf.subnet_id, worker_on_master=job_conf.worker_on_master) #TODO: utils.print_job_conf(job_configuration) spark_client.submit_job(job_configuration)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) job_conf = JobConfig() job_conf.merge(args.job_id, args.job_conf) # by default, load spark configuration files in .aztk/ spark_configuration = config.load_aztk_spark_config() # overwrite with values in job_conf if they exist if job_conf.spark_defaults_conf: spark_configuration.spark_defaults_conf = job_conf.spark_defaults_conf if job_conf.spark_env_sh: spark_configuration.spark_env_sh = job_conf.spark_env_sh if job_conf.core_site_xml: spark_configuration.core_site_xml = job_conf.core_site_xml job_configuration = aztk.spark.models.JobConfiguration( id=job_conf.id, applications=job_conf.applications, spark_configuration=spark_configuration, vm_size=job_conf.vm_size, toolkit=job_conf.toolkit, max_dedicated_nodes=job_conf.max_dedicated_nodes, max_low_pri_nodes=job_conf.max_low_pri_nodes, subnet_id=job_conf.subnet_id, plugins=job_conf.plugins, worker_on_master=job_conf.worker_on_master, scheduling_target=job_conf.scheduling_target, ) # TODO: utils.print_job_conf(job_configuration) spark_client.job.submit(job_configuration, args.vm_os_ver)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if spark_client.stop_job_app(args.job_id, args.app_name): log.info("Stopped app {0}".format(args.app_name)) else: log.error("App with name {0} does not exist or was already deleted")
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) log.info("-------------------------------------------") log.info("spark cluster id: %s", args.cluster_id) log.info("username: %s", args.username) log.info("-------------------------------------------") if args.ssh_key: ssh_key = args.ssh_key else: ssh_key = spark_client.secrets_configuration.ssh_pub_key ssh_key, password = utils.get_ssh_key_or_prompt( ssh_key, args.username, args.password, spark_client.secrets_configuration) spark_client.cluster.create_user(id=args.cluster_id, username=args.username, password=password, ssh_key=ssh_key) if password: log.info("password: %s", "*" * len(password)) elif ssh_key: log.info("ssh public key: %s", ssh_key) log.info("-------------------------------------------")
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) timestr = time.strftime("%Y%m%d-%H%M%S") if not args.output: args.output = os.path.join(os.getcwd(), "debug-{0}-{1}".format(args.cluster_id, timestr)) with utils.Spinner(): spark_client.cluster.diagnostics(id=args.cluster_id, output_directory=args.output, brief=args.brief)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if spark_client.job.stop_application(args.job_id, args.app_name): log.info("Stopped app %s", args.app_name) else: log.error("App with name %s does not exist or was already deleted", args.app_name)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) with utils.Spinner(): copy_output = spark_client.cluster.copy( id=args.cluster_id, source_path=args.source_path, destination_path=args.dest_path, internal=args.internal) for node_output in copy_output: utils.log_node_copy_output(node_output) sys.exit(0 if not any([node_output.error for node_output in copy_output]) else 1)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_conf = ClusterConfiguration() cluster_conf.spark_configuration = load_aztk_spark_config(args.spark_conf) # read cluster.yaml configuration file, overwrite values with args file_config, wait = config.read_cluster_config() if args.cluster_path is None \ else config.read_cluster_config(args.cluster_path) cluster_conf.merge(file_config) cluster_conf.merge( ClusterConfiguration( cluster_id=args.cluster_id, size=args.size, size_low_priority=args.size_low_priority, vm_size=args.vm_size, subnet_id=args.subnet_id, user_configuration=UserConfiguration(username=args.username, password=args.password), )) if cluster_conf.toolkit: if args.docker_repo: cluster_conf.toolkit.docker_repo = args.docker_repo if args.docker_run_options: cluster_conf.toolkit.docker_run_options = args.docker_run_options wait = wait if args.wait is None else args.wait user_configuration = cluster_conf.user_configuration if user_configuration and user_configuration.username: ssh_key, password = utils.get_ssh_key_or_prompt( spark_client.secrets_configuration.ssh_pub_key, user_configuration.username, user_configuration.password, spark_client.secrets_configuration, ) cluster_conf.user_configuration = aztk.spark.models.UserConfiguration( username=user_configuration.username, password=password, ssh_key=ssh_key) else: cluster_conf.user_configuration = None cluster_conf.validate() utils.print_cluster_conf(cluster_conf, wait) with utils.Spinner(): # create spark cluster cluster = spark_client.cluster.create( cluster_configuration=cluster_conf, vm_ver=args.vm_os_ver, wait=wait) if wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)
def execute(args: typing.NamedTuple): if not args.wait and args.output: raise aztk.error.AztkError("--output flag requires --wait flag") spark_client = aztk.spark.Client(config.load_aztk_secrets()) jars = [] py_files = [] files = [] if args.jars is not None: jars = args.jars.replace(" ", "").split(",") if args.py_files is not None: py_files = args.py_files.replace(" ", "").split(",") if args.files is not None: files = args.files.replace(" ", "").split(",") log_application(args, jars, py_files, files) spark_client.cluster.submit( id=args.cluster_id, application=aztk.spark.models.ApplicationConfiguration( name=args.name, application=args.app, application_args=args.app_args, main_class=args.main_class, jars=jars, py_files=py_files, files=files, driver_java_options=args.driver_java_options, driver_library_path=args.driver_library_path, driver_class_path=args.driver_class_path, driver_memory=args.driver_memory, executor_memory=args.executor_memory, driver_cores=args.driver_cores, executor_cores=args.executor_cores, max_retry_count=args.max_retry_count, ), remote=args.remote, internal=args.internal, wait=False, ) if args.wait: if not args.output: exit_code = utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.name) else: with utils.Spinner(): spark_client.cluster.wait(id=args.cluster_id, application_name=args.name) application_log = spark_client.cluster.get_application_log( id=args.cluster_id, application_name=args.name) with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(application_log.log) exit_code = application_log.exit_code sys.exit(exit_code)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) with utils.Spinner(): if args.node_id: results = [spark_client.node_run(args.cluster_id, args.node_id, args.command, args.host, args.internal)] else: results = spark_client.cluster_run(args.cluster_id, args.command, args.host, args.internal) [utils.log_execute_result(node_id, result) for node_id, result in results]
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster = spark_client.get_cluster(args.cluster_id) cluster_config = spark_client.get_cluster_config(args.cluster_id) ssh_conf = SshConfig() ssh_conf.merge(cluster_id=args.cluster_id, username=args.username, job_ui_port=args.jobui, job_history_ui_port=args.jobhistoryui, web_ui_port=args.webui, jupyter_port=args.jupyter, name_node_ui_port=args.namenodeui, rstudio_server_port=args.rstudioserver, host=args.host, connect=args.connect) log.info("-------------------------------------------") utils.log_property("spark cluster id", ssh_conf.cluster_id) utils.log_property("open webui", "{0}{1}".format(http_prefix, ssh_conf.web_ui_port)) utils.log_property("open jobui", "{0}{1}".format(http_prefix, ssh_conf.job_ui_port)) utils.log_property( "open jobhistoryui", "{0}{1}".format(http_prefix, ssh_conf.job_history_ui_port)) print_plugin_ports(cluster_config) utils.log_property("ssh username", ssh_conf.username) utils.log_property("connect", ssh_conf.connect) log.info("-------------------------------------------") # get ssh command try: ssh_cmd = utils.ssh_in_master( client=spark_client, cluster_id=ssh_conf.cluster_id, webui=ssh_conf.web_ui_port, jobui=ssh_conf.job_ui_port, jobhistoryui=ssh_conf.job_history_ui_port, username=ssh_conf.username, host=ssh_conf.host, connect=ssh_conf.connect) if not ssh_conf.connect: log.info("") log.info( "Use the following command to connect to your spark head node:" ) log.info("\t%s", ssh_cmd) except batch_error.BatchErrorException as e: if e.error.code == "PoolNotFound": raise aztk.error.AztkError( "The cluster you are trying to connect to does not exist.") else: raise
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_conf = ClusterConfiguration() cluster_conf.spark_configuration = load_aztk_spark_config() # read cluster.yaml configuration file, overwrite values with args file_config, wait = config.read_cluster_config() cluster_conf.merge(file_config) if args.size_low_pri is not None: deprecate("0.9.0", "--size-low-pri has been deprecated.", "Please use --size-low-priority.") args.size_low_priority = args.size_low_pri cluster_conf.merge(ClusterConfiguration( cluster_id=args.cluster_id, size=args.size, size_low_priority=args.size_low_priority, vm_size=args.vm_size, subnet_id=args.subnet_id, user_configuration=UserConfiguration( username=args.username, password=args.password, ))) if args.docker_repo and cluster_conf.toolkit: cluster_conf.toolkit.docker_repo = args.docker_repo wait = wait if args.wait is None else args.wait user_configuration = cluster_conf.user_configuration if user_configuration and user_configuration.username: ssh_key, password = utils.get_ssh_key_or_prompt(spark_client.secrets_config.ssh_pub_key, user_configuration.username, user_configuration.password, spark_client.secrets_config) cluster_conf.user_configuration = aztk.spark.models.UserConfiguration( username=user_configuration.username, password=password, ssh_key=ssh_key ) else: cluster_conf.user_configuration = None cluster_conf.validate() utils.print_cluster_conf(cluster_conf, wait) with utils.Spinner(): # create spark cluster cluster = spark_client.create_cluster( cluster_conf, wait=wait ) if wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) app_log = spark_client.get_job_application_log(args.job_id, args.app_name) if args.output: with utils.Spinner(): with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(app_log.log) else: log.print(app_log.log)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_id = args.cluster_id cluster = spark_client.get_cluster(cluster_id) utils.print_cluster(spark_client, cluster) configuration = spark_client.get_cluster_config(cluster_id) if configuration and args.show_config: log.info("-------------------------------------------") log.info("Cluster configuration:") utils.print_cluster_conf(configuration, False)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if args.tail: utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.app_name) else: app_logs = spark_client.get_application_log( cluster_id=args.cluster_id, application_name=args.app_name) print(app_logs.log)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_conf = ClusterConfiguration() cluster_conf.spark_configuration = load_aztk_spark_config() # read cluster.yaml configuartion file, overwrite values with args file_config, wait = config.read_cluster_config() cluster_conf.merge(file_config) cluster_conf.merge(ClusterConfiguration( cluster_id=args.cluster_id, vm_count=args.size, vm_low_pri_count=args.size_low_pri, vm_size=args.vm_size, subnet_id=args.subnet_id, user_configuration=UserConfiguration( username=args.username, password=args.password, ), docker_repo=args.docker_repo)) wait = wait if args.wait is None else args.wait user_configuration = cluster_conf.user_configuration if user_configuration and user_configuration.username: ssh_key, password = utils.get_ssh_key_or_prompt(spark_client.secrets_config.ssh_pub_key, user_configuration.username, user_configuration.password, spark_client.secrets_config) cluster_conf.user_configuration = aztk.spark.models.UserConfiguration( username=user_configuration.username, password=password, ssh_key=ssh_key ) else: cluster_conf.user_configuration = None utils.print_cluster_conf(cluster_conf, wait) spinner = utils.Spinner() spinner.start() # create spark cluster cluster = spark_client.create_cluster( cluster_conf, wait=wait ) spinner.stop() if wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) with utils.Spinner(): copy_output = spark_client.cluster_copy( cluster_id=args.cluster_id, source_path=args.source_path, destination_path=args.dest_path, internal=args.internal) [ print_copy_result(node_id, result, err) for node_id, result, err in copy_output ] sys.exit(0 if all([result for _, result, _ in copy_output]) else 1)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if args.tail: utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.app_name) else: app_log = spark_client.cluster.get_application_log(id=args.cluster_id, application_name=args.app_name) if args.output: with utils.Spinner(): with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(app_log.log) else: log.print(app_log.log)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) job_conf = JobConfig() job_conf.merge(args.job_id, args.job_conf) aztk_applications = [] for application in job_conf.applications: aztk_applications.append( aztk.spark.models.ApplicationConfiguration( name=application.get('name'), application=application.get('application'), application_args=application.get('application_args'), main_class=application.get('main_class'), jars=[], py_files=[], files=[], driver_java_options=application.get('driver_java_options'), driver_library_path=application.get('driver_library_path'), driver_class_path=application.get('driver_class_path'), driver_memory=application.get('driver_memory'), executor_memory=application.get('executor_memory'), driver_cores=application.get('driver_cores'), executor_cores=application.get('executor_cores'))) # by default, load spark configuration files in .aztk/ spark_configuration = config.load_aztk_spark_config() # overwrite with values in job_conf if they exist if job_conf.spark_defaults_conf: spark_configuration.spark_defaults_conf = job_conf.spark_defaults_conf if job_conf.spark_env_sh: spark_configuration.spark_env_sh = job_conf.spark_env_sh if job_conf.core_site_xml: spark_configuration.core_site_xml = job_conf.core_site_xml job_configuration = aztk.spark.models.JobConfiguration( id=job_conf.id, applications=aztk_applications, custom_scripts=job_conf.custom_scripts, spark_configuration=spark_configuration, vm_size=job_conf.vm_size, docker_repo=job_conf.docker_repo, max_dedicated_nodes=job_conf.max_dedicated_nodes, max_low_pri_nodes=job_conf.max_low_pri_nodes, subnet_id=job_conf.subnet_id, worker_on_master=job_conf.worker_on_master) #TODO: utils.print_job_conf(job_configuration) spark_client.submit_job(job_configuration)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_id = args.cluster_id if not args.force: confirmation_cluster_id = input("Please confirm the id of the cluster you wish to delete: ") if confirmation_cluster_id != cluster_id: log.error("Confirmation cluster id does not match. Please try again.") return if spark_client.delete_cluster(cluster_id): log.info("Deleting cluster %s", cluster_id) else: log.error("Cluster with id '%s' doesn't exist or was already deleted.", cluster_id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) job_id = args.job_id if not args.force: # check if job exists before prompting for confirmation spark_client.get_job(job_id) confirmation_cluster_id = input("Please confirm the id of the cluster you wish to delete: ") if confirmation_cluster_id != job_id: log.error("Confirmation cluster id does not match. Please try again.") return if spark_client.delete_job(job_id): log.info("Deleting Job %s", job_id) else: log.error("Job with id '%s' doesn't exist or was already deleted.", job_id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) utils.print_jobs(spark_client.list_jobs())
from azure.batch.models import BatchErrorException import aztk.spark import pytest from aztk.utils import constants from aztk.error import AztkError from aztk_cli import config # base cluster name dt = datetime.now() current_time = dt.microsecond base_cluster_id = "cluster-{}".format(current_time) # load secrets # note: this assumes secrets are set up in .aztk/secrets spark_client = aztk.spark.Client(config.load_aztk_secrets()) # helper method def wait_until_cluster_deleted(cluster_id: str): while True: try: spark_client.get_cluster(cluster_id) time.sleep(1) except AztkError: # break when the cluster is not found break def test_create_cluster(): test_id = "test-create-"
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) spark_client.stop_job(args.job_id) print("Stopped Job {0}".format(args.job_id))
def execute(args: typing.NamedTuple): if not args.wait and args.output: raise aztk.error.AztkError("--output flag requires --wait flag") spark_client = aztk.spark.Client(config.load_aztk_secrets()) jars = [] py_files = [] files = [] if args.jars is not None: jars = args.jars.replace(' ', '').split(',') if args.py_files is not None: py_files = args.py_files.replace(' ', '').split(',') if args.files is not None: files = args.files.replace(' ', '').split(',') log.info("-------------------------------------------") log.info("Spark cluster id: %s", args.cluster_id) log.info("Spark app name: %s", args.name) log.info("Wait for app completion: %s", args.wait) if args.main_class is not None: log.info("Entry point class: %s", args.main_class) if jars: log.info("JARS: %s", jars) if py_files: log.info("PY_Files: %s", py_files) if files: log.info("Files: %s", files) if args.driver_java_options is not None: log.info("Driver java options: %s", args.driver_java_options) if args.driver_library_path is not None: log.info("Driver library path: %s", args.driver_library_path) if args.driver_class_path is not None: log.info("Driver class path: %s", args.driver_class_path) if args.driver_memory is not None: log.info("Driver memory: %s", args.driver_memory) if args.executor_memory is not None: log.info("Executor memory: %s", args.executor_memory) if args.driver_cores is not None: log.info("Driver cores: %s", args.driver_cores) if args.executor_cores is not None: log.info("Executor cores: %s", args.executor_cores) log.info("Application: %s", args.app) log.info("Application arguments: %s", args.app_args) log.info("-------------------------------------------") spark_client.submit(cluster_id=args.cluster_id, application=aztk.spark.models.ApplicationConfiguration( name=args.name, application=args.app, application_args=args.app_args, main_class=args.main_class, jars=jars, py_files=py_files, files=files, driver_java_options=args.driver_java_options, driver_library_path=args.driver_library_path, driver_class_path=args.driver_class_path, driver_memory=args.driver_memory, executor_memory=args.executor_memory, driver_cores=args.driver_cores, executor_cores=args.executor_cores, max_retry_count=args.max_retry_count), remote=args.remote, wait=False) if args.wait: if not args.output: exit_code = utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.name) else: with utils.Spinner(): spark_client.wait_until_application_done( cluster_id=args.cluster_id, task_id=args.name) application_log = spark_client.get_application_log( cluster_id=args.cluster_id, application_name=args.name) with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(application_log.log) exit_code = application_log.exit_code sys.exit(exit_code)