def wait_until_application_done(self, cluster_id: str, task_id: str):
try:
helpers.wait_for_task_to_complete(job_id=cluster_id,
task_id=task_id,
batch_client=self.batch_client)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def submit_application(
core_cluster_operations,
spark_cluster_operations,
cluster_id,
application,
remote: bool = False,
wait: bool = False,
internal: bool = False,
):
"""
Submit a spark app
"""
task = spark_cluster_operations._generate_application_task(
core_cluster_operations, cluster_id, application, remote)
task = affinitize_task_to_master(core_cluster_operations,
spark_cluster_operations, cluster_id,
task)
scheduling_target = get_cluster_scheduling_target(core_cluster_operations,
cluster_id)
if scheduling_target is not models.SchedulingTarget.Any:
schedule_with_target(core_cluster_operations, spark_cluster_operations,
cluster_id, scheduling_target, task, wait,
internal)
else:
# Add task to batch job (which has the same name as cluster_id)
core_cluster_operations.batch_client.task.add(job_id=cluster_id,
task=task)
if wait:
helpers.wait_for_task_to_complete(
job_id=cluster_id,
task_id=task.id,
batch_client=core_cluster_operations.batch_client)
def submit_application(spark_client, cluster_id, application, wait: bool = False):
"""
Submit a spark app
"""
task = generate_task(spark_client, cluster_id, application)
task = affinitize_task_to_master(spark_client, cluster_id, task)
# Add task to batch job (which has the same name as cluster_id)
job_id = cluster_id
spark_client.batch_client.task.add(job_id=job_id, task=task)
if wait:
helpers.wait_for_task_to_complete(job_id=job_id, task_id=task.id, batch_client=spark_client.batch_client)
def submit_application(core_cluster_operations,
spark_cluster_operations,
cluster_id,
application,
remote: bool = False,
wait: bool = False):
"""
Submit a spark app
"""
task = spark_cluster_operations._generate_application_task(core_cluster_operations, cluster_id, application, remote)
task = affinitize_task_to_master(core_cluster_operations, spark_cluster_operations, cluster_id, task)
# Add task to batch job (which has the same name as cluster_id)
job_id = cluster_id
core_cluster_operations.batch_client.task.add(job_id=job_id, task=task)
if wait:
helpers.wait_for_task_to_complete(
job_id=job_id, task_id=task.id, batch_client=core_cluster_operations.batch_client)
def submit_application(spark_client, cluster_id, application, wait: bool = False):
"""
Submit a spark app
"""
resource_files = []
app_resource_file = helpers.upload_file_to_container(container_name=application.name,
file_path=application.application,
blob_client=spark_client.blob_client,
use_full_path=False)
# Upload application file
resource_files.append(app_resource_file)
# Upload dependent JARS
jar_resource_file_paths = []
for jar in application.jars:
current_jar_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=jar,
blob_client=spark_client.blob_client,
use_full_path=False)
jar_resource_file_paths.append(current_jar_resource_file_path)
resource_files.append(current_jar_resource_file_path)
# Upload dependent python files
py_files_resource_file_paths = []
for py_file in application.py_files:
current_py_files_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=py_file,
blob_client=spark_client.blob_client,
use_full_path=False)
py_files_resource_file_paths.append(
current_py_files_resource_file_path)
resource_files.append(current_py_files_resource_file_path)
# Upload other dependent files
files_resource_file_paths = []
for file in application.files:
files_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=file,
blob_client=spark_client.blob_client,
use_full_path=False)
files_resource_file_paths.append(files_resource_file_path)
resource_files.append(files_resource_file_path)
# create command to submit task
cmd = __app_submit_cmd(
spark_client=spark_client,
cluster_id=cluster_id,
name=application.name,
app=app_resource_file.file_path,
app_args=application.application_args,
main_class=application.main_class,
jars=[jar_resource_file_path.file_path for jar_resource_file_path in jar_resource_file_paths],
py_files=[py_files_resource.file_path for py_files_resource in py_files_resource_file_paths],
files=[file_resource_file_path.file_path for file_resource_file_path in files_resource_file_paths],
driver_java_options=application.driver_java_options,
driver_library_path=application.driver_library_path,
driver_class_path=application.driver_class_path,
driver_memory=application.driver_memory,
executor_memory=application.executor_memory,
driver_cores=application.driver_cores,
executor_cores=application.executor_cores)
# Get cluster size
cluster = spark_client.get_cluster(cluster_id)
# Affinitize task to master node
# master_node_affinity_id = helpers.get_master_node_id(cluster_id, spark_client.batch_client)
rls = spark_client.get_remote_login_settings(cluster.id, cluster.master_node_id)
# Create task
task = batch_models.TaskAddParameter(
id=application.name,
affinity_info=batch_models.AffinityInformation(
affinity_id=cluster.master_node_id),
command_line=helpers.wrap_commands_in_shell(cmd),
resource_files=resource_files,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.task,
elevation_level=batch_models.ElevationLevel.admin))
)
# Add task to batch job (which has the same name as cluster_id)
job_id = cluster_id
spark_client.batch_client.task.add(job_id=job_id, task=task)
if wait:
helpers.wait_for_task_to_complete(job_id=job_id, task_id=task.id, batch_client=spark_client.batch_client)
