def get_log(base_operations,
cluster_id: str,
application_name: str,
tail=False,
current_bytes: int = 0):
job_id = cluster_id
task_id = application_name
cluster_configuration = base_operations.get_cluster_configuration(
cluster_id)
if cluster_configuration.scheduling_target is not models.SchedulingTarget.Any:
task = wait_for_scheduling_target_task(base_operations, cluster_id,
application_name)
return get_log_from_storage(base_operations.blob_client, cluster_id,
application_name, task)
else:
task = __wait_for_app_to_be_running(base_operations, cluster_id,
application_name)
if not __check_task_node_exist(base_operations.batch_client,
cluster_id, task):
return get_log_from_storage(base_operations.blob_client,
cluster_id, application_name, task)
file = __get_output_file_properties(base_operations.batch_client,
cluster_id, application_name)
target_bytes = file.content_length
if target_bytes != current_bytes:
ocp_range = None
if tail:
ocp_range = "bytes={0}-{1}".format(current_bytes, target_bytes - 1)
stream = base_operations.batch_client.file.get_from_task(
job_id, task_id, output_file,
batch_models.FileGetFromTaskOptions(ocp_range=ocp_range))
content = helpers.read_stream_as_string(stream)
return models.ApplicationLog(
name=application_name,
cluster_id=cluster_id,
application_state=task.state,
log=content,
total_bytes=target_bytes,
exit_code=task.exit_code,
)
else:
return models.ApplicationLog(
name=application_name,
cluster_id=cluster_id,
application_state=task.state,
log="",
total_bytes=target_bytes,
exit_code=task.exit_code,
)
def get_log_from_storage(blob_client, container_name, application_name, task):
"""
Args:
blob_client (:obj:`azure.storage.blob.BlockBlobService`): Client used to interact with the Azure Storage
Blob service.
container_name (:obj:`str`): the name of the Azure Blob storage container to get data from
application_name (:obj:`str`): the name of the application to get logs for
task (:obj:`aztk.models.Task`): the aztk task for for this application
"""
try:
blob = blob_client.get_blob_to_text(
container_name,
application_name + "/" + constants.SPARK_SUBMIT_LOGS_FILE)
except azure.common.AzureMissingResourceHttpError:
raise error.AztkError(
"Logs not found in your storage account. They were either deleted or never existed."
)
return models.ApplicationLog(
name=application_name,
cluster_id=container_name,
application_state=task.state,
log=blob.content,
total_bytes=blob.properties.content_length,
exit_code=task.exit_code,
)
def get_log(batch_client,
blob_client,
cluster_id: str,
application_name: str,
tail=False,
current_bytes: int = 0):
job_id = cluster_id
task_id = application_name
task = __wait_for_app_to_be_running(batch_client, cluster_id,
application_name)
if not __check_task_node_exist(batch_client, cluster_id, task):
return get_log_from_storage(blob_client, cluster_id, application_name,
task)
file = __get_output_file_properties(batch_client, cluster_id,
application_name)
target_bytes = file.content_length
if target_bytes != current_bytes:
ocp_range = None
if tail:
ocp_range = "bytes={0}-{1}".format(current_bytes, target_bytes - 1)
stream = batch_client.file.get_from_task(
job_id, task_id, output_file,
batch_models.FileGetFromTaskOptions(ocp_range=ocp_range))
content = helpers.read_stream_as_string(stream)
return models.ApplicationLog(name=application_name,
cluster_id=cluster_id,
application_state=task.state._value_,
log=content,
total_bytes=target_bytes,
exit_code=task.execution_info.exit_code)
else:
return models.ApplicationLog(name=application_name,
cluster_id=cluster_id,
application_state=task.state._value_,
log='',
total_bytes=target_bytes,
exit_code=task.execution_info.exit_code)
def get_log_from_storage(blob_client, container_name, application_name, task):
try:
blob = blob_client.get_blob_to_text(
container_name,
application_name + '/' + constants.SPARK_SUBMIT_LOGS_FILE)
except azure.common.AzureMissingResourceHttpError:
raise error.AztkError(
"Logs not found in your storage account. They were either deleted or never existed."
)
return models.ApplicationLog(name=application_name,
cluster_id=container_name,
application_state=task.state._value_,
log=blob.content,
total_bytes=blob.properties.content_length,
exit_code=task.execution_info.exit_code)