def ConstructSparkService(self): """Create the spark_service object and create groups for its vms.""" if self.config.spark_service is None: return spark_spec = self.config.spark_service # Worker group is required, master group is optional cloud = spark_spec.worker_group.cloud if spark_spec.master_group: cloud = spark_spec.master_group.cloud providers.LoadProvider(cloud) service_type = spark_spec.service_type spark_service_class = spark_service.GetSparkServiceClass( cloud, service_type) self.spark_service = spark_service_class(spark_spec) # If this is Pkb managed, the benchmark spec needs to adopt vms. if service_type == spark_service.PKB_MANAGED: for name, group_spec in [('master_group', spark_spec.master_group), ('worker_group', spark_spec.worker_group) ]: if name in self.vms_to_boot: raise Exception( 'Cannot have a vm group {0} with a {1} spark ' 'service'.format(name, spark_service.PKB_MANAGED)) self.vms_to_boot[name] = group_spec
def ConstructSparkService(self): if self.config.spark_service is None: return providers.LoadProvider(self.config.spark_service.cloud) spark_service_spec = self.config.spark_service service_type = spark_service_spec.service_type spark_service_class = spark_service.GetSparkServiceClass( spark_service_spec.cloud, service_type) self.spark_service = spark_service_class(spark_service_spec)
def ConstructSparkService(self): if self.config.spark_service is None: return providers.LoadProvider(self.config.spark_service.cloud) spark_service_spec = self.config.spark_service service_type = spark_service_spec.spark_service_type spark_service_class = spark_service.GetSparkServiceClass( spark_service_spec.cloud, service_type) if self.config.spark_service.static_cluster_name: name = self.config.spark_service.static_cluster_name static_cluster = True else: name = 'pkb-' + FLAGS.run_uri static_cluster = False self.spark_service = spark_service_class(name, static_cluster, spark_service_spec)