def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if args.tail: utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.app_name) else: app_logs = spark_client.get_application_log( cluster_id=args.cluster_id, application_name=args.app_name) print(app_logs.log)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) if args.tail: utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.app_name) else: app_log = spark_client.cluster.get_application_log(id=args.cluster_id, application_name=args.app_name) if args.output: with utils.Spinner(): with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(app_log.log) else: log.print(app_log.log)
def execute(args: typing.NamedTuple): if not args.wait and args.output: raise aztk.error.AztkError("--output flag requires --wait flag") spark_client = aztk.spark.Client(config.load_aztk_secrets()) jars = [] py_files = [] files = [] if args.jars is not None: jars = args.jars.replace(" ", "").split(",") if args.py_files is not None: py_files = args.py_files.replace(" ", "").split(",") if args.files is not None: files = args.files.replace(" ", "").split(",") log_application(args, jars, py_files, files) spark_client.cluster.submit( id=args.cluster_id, application=aztk.spark.models.ApplicationConfiguration( name=args.name, application=args.app, application_args=args.app_args, main_class=args.main_class, jars=jars, py_files=py_files, files=files, driver_java_options=args.driver_java_options, driver_library_path=args.driver_library_path, driver_class_path=args.driver_class_path, driver_memory=args.driver_memory, executor_memory=args.executor_memory, driver_cores=args.driver_cores, executor_cores=args.executor_cores, max_retry_count=args.max_retry_count, ), remote=args.remote, internal=args.internal, wait=False, ) if args.wait: if not args.output: exit_code = utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.name) else: with utils.Spinner(): spark_client.cluster.wait(id=args.cluster_id, application_name=args.name) application_log = spark_client.cluster.get_application_log( id=args.cluster_id, application_name=args.name) with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(application_log.log) exit_code = application_log.exit_code sys.exit(exit_code)
def execute(args: typing.NamedTuple): if not args.wait and args.output: raise aztk.error.AztkError("--output flag requires --wait flag") spark_client = aztk.spark.Client(config.load_aztk_secrets()) jars = [] py_files = [] files = [] if args.jars is not None: jars = args.jars.replace(' ', '').split(',') if args.py_files is not None: py_files = args.py_files.replace(' ', '').split(',') if args.files is not None: files = args.files.replace(' ', '').split(',') log.info("-------------------------------------------") log.info("Spark cluster id: %s", args.cluster_id) log.info("Spark app name: %s", args.name) log.info("Wait for app completion: %s", args.wait) if args.main_class is not None: log.info("Entry point class: %s", args.main_class) if jars: log.info("JARS: %s", jars) if py_files: log.info("PY_Files: %s", py_files) if files: log.info("Files: %s", files) if args.driver_java_options is not None: log.info("Driver java options: %s", args.driver_java_options) if args.driver_library_path is not None: log.info("Driver library path: %s", args.driver_library_path) if args.driver_class_path is not None: log.info("Driver class path: %s", args.driver_class_path) if args.driver_memory is not None: log.info("Driver memory: %s", args.driver_memory) if args.executor_memory is not None: log.info("Executor memory: %s", args.executor_memory) if args.driver_cores is not None: log.info("Driver cores: %s", args.driver_cores) if args.executor_cores is not None: log.info("Executor cores: %s", args.executor_cores) log.info("Application: %s", args.app) log.info("Application arguments: %s", args.app_args) log.info("-------------------------------------------") spark_client.submit(cluster_id=args.cluster_id, application=aztk.spark.models.ApplicationConfiguration( name=args.name, application=args.app, application_args=args.app_args, main_class=args.main_class, jars=jars, py_files=py_files, files=files, driver_java_options=args.driver_java_options, driver_library_path=args.driver_library_path, driver_class_path=args.driver_class_path, driver_memory=args.driver_memory, executor_memory=args.executor_memory, driver_cores=args.driver_cores, executor_cores=args.executor_cores, max_retry_count=args.max_retry_count), remote=args.remote, wait=False) if args.wait: if not args.output: exit_code = utils.stream_logs(client=spark_client, cluster_id=args.cluster_id, application_name=args.name) else: with utils.Spinner(): spark_client.wait_until_application_done( cluster_id=args.cluster_id, task_id=args.name) application_log = spark_client.get_application_log( cluster_id=args.cluster_id, application_name=args.name) with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(application_log.log) exit_code = application_log.exit_code sys.exit(exit_code)