def create_cluster(self,
cluster_conf: models.ClusterConfiguration,
wait: bool = False):
try:
zip_resource_files = upload_node_scripts.zip_scripts(
self.blob_client, cluster_conf.cluster_id,
cluster_conf.custom_scripts, cluster_conf.spark_configuration,
cluster_conf.user_configuration)
start_task = create_cluster_helper.generate_cluster_start_task(
self, zip_resource_files, cluster_conf.gpu_enabled,
cluster_conf.docker_repo, cluster_conf.file_shares)
software_metadata_key = "spark"
vm_image = models.VmImage(publisher='Canonical',
offer='UbuntuServer',
sku='16.04')
cluster = self.__create_pool_and_job(cluster_conf,
software_metadata_key,
start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(self, cluster.id)
cluster = self.get_cluster(cluster.id)
return cluster
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def wait_until_cluster_is_ready(self, cluster_id: str):
try:
util.wait_for_master_to_be_ready(self, cluster_id)
pool = self.batch_client.pool.get(cluster_id)
nodes = self.batch_client.compute_node.list(pool_id=cluster_id)
return models.Cluster(pool, nodes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(core_cluster_operations,
spark_cluster_operations,
cluster_conf: models.ClusterConfiguration,
vm_image: base_models.VmImage,
wait: bool = False):
"""
Create a new aztk spark cluster
Args:
cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
wait(bool): If you should wait for the cluster to be ready before returning
vm_image: models for cluster vm
Returns:
:obj:`aztk.spark.models.Cluster`
"""
cluster_conf = _apply_default_for_cluster_config(cluster_conf)
cluster_conf.validate()
cluster_data = core_cluster_operations.get_cluster_data(
cluster_conf.cluster_id)
try:
zip_resource_files = None
node_data = NodeData(cluster_conf).add_core().done()
zip_resource_files = cluster_data.upload_node_data(
node_data).to_resource_file()
start_task = spark_cluster_operations._generate_cluster_start_task(
core_cluster_operations,
zip_resource_files,
cluster_conf.cluster_id,
cluster_conf.gpu_enabled(),
cluster_conf.get_docker_repo(),
cluster_conf.get_docker_run_options(),
cluster_conf.file_shares,
cluster_conf.mixed_mode(),
cluster_conf.worker_on_master,
)
software_metadata_key = base_models.Software.spark
cluster = core_cluster_operations.create(cluster_conf,
software_metadata_key,
start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(core_cluster_operations,
spark_cluster_operations,
cluster.id)
cluster = spark_cluster_operations.get(cluster.id)
return cluster
except BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(self, configuration: models.ClusterConfiguration, wait: bool = False):
"""
Create a new aztk spark cluster
Args:
cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
wait(bool): If you should wait for the cluster to be ready before returning
Returns:
aztk.spark.models.Cluster
"""
cluster_conf = models.ClusterConfiguration()
cluster_conf.merge(DEFAULT_CLUSTER_CONFIG)
cluster_conf.merge(configuration)
cluster_conf.validate()
cluster_data = self._get_cluster_data(cluster_conf.cluster_id)
try:
zip_resource_files = None
node_data = NodeData(cluster_conf).add_core().done()
zip_resource_files = cluster_data.upload_node_data(node_data).to_resource_file()
start_task = create_cluster_helper.generate_cluster_start_task(self,
zip_resource_files,
cluster_conf.cluster_id,
cluster_conf.gpu_enabled(),
cluster_conf.get_docker_repo(),
cluster_conf.file_shares,
cluster_conf.plugins,
cluster_conf.mixed_mode(),
cluster_conf.worker_on_master)
software_metadata_key = "spark"
vm_image = models.VmImage(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04')
cluster = self.__create_pool_and_job(
cluster_conf, software_metadata_key, start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(self, cluster.id)
cluster = self.get_cluster(cluster.id)
return cluster
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(self,
cluster_conf: models.ClusterConfiguration,
wait: bool = False):
cluster_conf.validate()
cluster_data = self._get_cluster_data(cluster_conf.cluster_id)
try:
zip_resource_files = None
node_data = NodeData(cluster_conf).add_core().done()
zip_resource_files = cluster_data.upload_node_data(
node_data).to_resource_file()
start_task = create_cluster_helper.generate_cluster_start_task(
self, zip_resource_files, cluster_conf.gpu_enabled(),
cluster_conf.docker_repo,
cluster_conf.file_shares, cluster_conf.plugins,
cluster_conf.mixed_mode(), cluster_conf.worker_on_master)
software_metadata_key = "spark"
vm_image = models.VmImage(publisher='Canonical',
offer='UbuntuServer',
sku='16.04')
cluster = self.__create_pool_and_job(cluster_conf,
software_metadata_key,
start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(self, cluster.id)
cluster = self.get_cluster(cluster.id)
return cluster
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
