def _Create(self):
"""Create an un-managed yarn cluster."""
logging.info('Should have created vms by now.')
logging.info(str(self.vms))
def InstallSpark(vm):
vm.Install('spark')
vm_util.RunThreaded(
InstallSpark, self.vms['worker_group'] + self.vms['master_group'])
self.leader = self.vms['master_group'][0]
spark.ConfigureAndStart(self.leader, self.vms['worker_group'])
def _Create(self):
"""Create an un-managed yarn cluster."""
logging.info('Should have created vms by now.')
logging.info(str(self.vms))
def InstallSpark(vm):
vm.Install('spark')
if self.cloud == 'GCP':
hadoop.InstallGcsConnector(vm)
if self.cloud == 'AWS':
hadoop.InstallS3Connector(vm)
vm_util.RunThreaded(
InstallSpark, self.vms['worker_group'] + self.vms['master_group'])
self.leader = self.vms['master_group'][0]
spark.ConfigureAndStart(self.leader,
self.vms['worker_group'],
configure_s3=self.cloud == 'AWS')
def _Create(self):
"""Create an un-managed yarn cluster."""
logging.info('Should have created vms by now.')
logging.info(str(self.vms))
def InstallSpark(vm):
vm.Install('spark')
if self.cloud == 'GCP':
hadoop.InstallGcsConnector(vm)
if 'worker_group' not in self.vms:
raise errors.Resource.CreationError(
'UnmanagedDpbSparkCluster requires VMs in a worker_group.')
vm_util.RunThreaded(
InstallSpark, self.vms['worker_group'] + self.vms['master_group'])
self.leader = self.vms['master_group'][0]
spark.ConfigureAndStart(self.leader,
self.vms['worker_group'],
configure_s3=self.cloud == 'AWS')