def ConstructSparkService(self):
"""Create the spark_service object and create groups for its vms."""
if self.config.spark_service is None:
return
spark_spec = self.config.spark_service
# Worker group is required, master group is optional
cloud = spark_spec.worker_group.cloud
if spark_spec.master_group:
cloud = spark_spec.master_group.cloud
providers.LoadProvider(cloud)
service_type = spark_spec.service_type
spark_service_class = spark_service.GetSparkServiceClass(
cloud, service_type)
self.spark_service = spark_service_class(spark_spec)
# If this is Pkb managed, the benchmark spec needs to adopt vms.
if service_type == spark_service.PKB_MANAGED:
for name, group_spec in [('master_group', spark_spec.master_group),
('worker_group', spark_spec.worker_group)
]:
if name in self.vms_to_boot:
raise Exception(
'Cannot have a vm group {0} with a {1} spark '
'service'.format(name, spark_service.PKB_MANAGED))
self.vms_to_boot[name] = group_spec
def ConstructSparkService(self):
if self.config.spark_service is None:
return
providers.LoadProvider(self.config.spark_service.cloud)
spark_service_spec = self.config.spark_service
service_type = spark_service_spec.service_type
spark_service_class = spark_service.GetSparkServiceClass(
spark_service_spec.cloud, service_type)
self.spark_service = spark_service_class(spark_service_spec)
def ConstructSparkService(self):
if self.config.spark_service is None:
return
providers.LoadProvider(self.config.spark_service.cloud)
spark_service_spec = self.config.spark_service
service_type = spark_service_spec.spark_service_type
spark_service_class = spark_service.GetSparkServiceClass(
spark_service_spec.cloud, service_type)
if self.config.spark_service.static_cluster_name:
name = self.config.spark_service.static_cluster_name
static_cluster = True
else:
name = 'pkb-' + FLAGS.run_uri
static_cluster = False
self.spark_service = spark_service_class(name, static_cluster,
spark_service_spec)
