def create_cluster(self, cluster_conf: models.ClusterConfiguration, wait: bool = False): try: zip_resource_files = upload_node_scripts.zip_scripts( self.blob_client, cluster_conf.cluster_id, cluster_conf.custom_scripts, cluster_conf.spark_configuration, cluster_conf.user_configuration) start_task = create_cluster_helper.generate_cluster_start_task( self, zip_resource_files, cluster_conf.gpu_enabled, cluster_conf.docker_repo, cluster_conf.file_shares) software_metadata_key = "spark" vm_image = models.VmImage(publisher='Canonical', offer='UbuntuServer', sku='16.04') cluster = self.__create_pool_and_job(cluster_conf, software_metadata_key, start_task, vm_image) # Wait for the master to be ready if wait: util.wait_for_master_to_be_ready(self, cluster.id) cluster = self.get_cluster(cluster.id) return cluster except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def wait_until_cluster_is_ready(self, cluster_id: str): try: util.wait_for_master_to_be_ready(self, cluster_id) pool = self.batch_client.pool.get(cluster_id) nodes = self.batch_client.compute_node.list(pool_id=cluster_id) return models.Cluster(pool, nodes) except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(core_cluster_operations, spark_cluster_operations, cluster_conf: models.ClusterConfiguration, vm_image: base_models.VmImage, wait: bool = False): """ Create a new aztk spark cluster Args: cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created wait(bool): If you should wait for the cluster to be ready before returning vm_image: models for cluster vm Returns: :obj:`aztk.spark.models.Cluster` """ cluster_conf = _apply_default_for_cluster_config(cluster_conf) cluster_conf.validate() cluster_data = core_cluster_operations.get_cluster_data( cluster_conf.cluster_id) try: zip_resource_files = None node_data = NodeData(cluster_conf).add_core().done() zip_resource_files = cluster_data.upload_node_data( node_data).to_resource_file() start_task = spark_cluster_operations._generate_cluster_start_task( core_cluster_operations, zip_resource_files, cluster_conf.cluster_id, cluster_conf.gpu_enabled(), cluster_conf.get_docker_repo(), cluster_conf.get_docker_run_options(), cluster_conf.file_shares, cluster_conf.mixed_mode(), cluster_conf.worker_on_master, ) software_metadata_key = base_models.Software.spark cluster = core_cluster_operations.create(cluster_conf, software_metadata_key, start_task, vm_image) # Wait for the master to be ready if wait: util.wait_for_master_to_be_ready(core_cluster_operations, spark_cluster_operations, cluster.id) cluster = spark_cluster_operations.get(cluster.id) return cluster except BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(self, configuration: models.ClusterConfiguration, wait: bool = False): """ Create a new aztk spark cluster Args: cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created wait(bool): If you should wait for the cluster to be ready before returning Returns: aztk.spark.models.Cluster """ cluster_conf = models.ClusterConfiguration() cluster_conf.merge(DEFAULT_CLUSTER_CONFIG) cluster_conf.merge(configuration) cluster_conf.validate() cluster_data = self._get_cluster_data(cluster_conf.cluster_id) try: zip_resource_files = None node_data = NodeData(cluster_conf).add_core().done() zip_resource_files = cluster_data.upload_node_data(node_data).to_resource_file() start_task = create_cluster_helper.generate_cluster_start_task(self, zip_resource_files, cluster_conf.cluster_id, cluster_conf.gpu_enabled(), cluster_conf.get_docker_repo(), cluster_conf.file_shares, cluster_conf.plugins, cluster_conf.mixed_mode(), cluster_conf.worker_on_master) software_metadata_key = "spark" vm_image = models.VmImage( publisher='Canonical', offer='UbuntuServer', sku='16.04') cluster = self.__create_pool_and_job( cluster_conf, software_metadata_key, start_task, vm_image) # Wait for the master to be ready if wait: util.wait_for_master_to_be_ready(self, cluster.id) cluster = self.get_cluster(cluster.id) return cluster except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(self, cluster_conf: models.ClusterConfiguration, wait: bool = False): cluster_conf.validate() cluster_data = self._get_cluster_data(cluster_conf.cluster_id) try: zip_resource_files = None node_data = NodeData(cluster_conf).add_core().done() zip_resource_files = cluster_data.upload_node_data( node_data).to_resource_file() start_task = create_cluster_helper.generate_cluster_start_task( self, zip_resource_files, cluster_conf.gpu_enabled(), cluster_conf.docker_repo, cluster_conf.file_shares, cluster_conf.plugins, cluster_conf.mixed_mode(), cluster_conf.worker_on_master) software_metadata_key = "spark" vm_image = models.VmImage(publisher='Canonical', offer='UbuntuServer', sku='16.04') cluster = self.__create_pool_and_job(cluster_conf, software_metadata_key, start_task, vm_image) # Wait for the master to be ready if wait: util.wait_for_master_to_be_ready(self, cluster.id) cluster = self.get_cluster(cluster.id) return cluster except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))