def create_cluster(self,
cluster_conf: models.ClusterConfiguration,
wait: bool = False):
try:
zip_resource_files = upload_node_scripts.zip_scripts(
self.blob_client, cluster_conf.custom_scripts,
cluster_conf.spark_configuration)
start_task = create_cluster_helper.generate_cluster_start_task(
self, zip_resource_files, cluster_conf.docker_repo)
software_metadata_key = "spark"
vm_image = models.VmImage(publisher='Canonical',
offer='UbuntuServer',
sku='16.04')
cluster = self.__create_pool_and_job(cluster_conf,
software_metadata_key,
start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(self, cluster.id)
cluster = self.get_cluster(cluster.id)
return cluster
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def wait_until_application_done(self, cluster_id: str, task_id: str):
try:
helpers.wait_for_task_to_complete(job_id=cluster_id,
task_id=task_id,
batch_client=self.batch_client)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def list_clusters(self):
try:
return [
models.Cluster(pool) for pool in self.__list_clusters(
aztk_sdk.models.Software.spark)
]
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def wait_until_cluster_is_ready(self, cluster_id: str):
try:
util.wait_for_master_to_be_ready(self, cluster_id)
pool = self.batch_client.pool.get(cluster_id)
nodes = self.batch_client.compute_node.list(pool_id=cluster_id)
return models.Cluster(pool, nodes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def submit(self,
cluster_id: str,
application: models.AppModel,
wait: bool = False):
try:
submit_helper.submit_application(self, cluster_id, application,
wait)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_application_log(self,
cluster_id: str,
application_name: str,
tail=False,
current_bytes: int = 0):
try:
return get_log_helper.get_log(self.batch_client, cluster_id,
application_name, tail,
current_bytes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_user(self,
cluster_id: str,
username: str,
password: str = None,
ssh_key: str = None) -> str:
try:
cluster = self.get_cluster(cluster_id)
master_node_id = cluster.master_node_id
self.__create_user(cluster.id, master_node_id, username, password,
ssh_key)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_remote_login_settings(self, cluster_id: str, node_id: str):
try:
return self.__get_remote_login_settings(cluster_id, node_id)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_cluster(self, cluster_id: str):
try:
pool, nodes = self.__get_pool_details(cluster_id)
return models.Cluster(pool, nodes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def delete_cluster(self, cluster_id: str):
try:
return self.__delete_pool_and_job(cluster_id)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_application_status(self, cluster_id: str, app_name: str):
try:
task = self.batch_client.task.get(cluster_id, app_name)
return task.state._value_
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
