def get_cluster_services(self, node_group=None):
node_processes = None
if node_group:
node_processes = node_group.node_processes
else:
node_processes = [np for ng in self.cluster.node_groups for np in ng.node_processes]
node_processes = g.unique_list(node_processes)
services = g.unique_list(node_processes, self.get_service)
return services + [swift.Swift()]
def get_cluster_services(self, node_group=None):
node_processes = None
if node_group:
node_processes = node_group.node_processes
else:
node_processes = [np for ng in self.cluster.node_groups
for np in ng.node_processes]
node_processes = g.unique_list(node_processes)
services = g.unique_list(node_processes, self.get_service)
return services + [swift.Swift()]
def _get_packages(self, node_processes):
result = []
result += self.dependencies
result += [(np.package, self.version)
for np in node_processes
if np != SPARK_HISTORY_SERVER]
return g.unique_list(result)
def get_open_ports(self, node_group):
result = []
for service in self.get_services():
for node_process in service.node_processes:
if node_process.ui_name in node_group.node_processes:
result += node_process.open_ports
return util.unique_list(result)
def get_configs(self):
if not self._configs:
configs = [c for s in self.get_services() for c in s.get_configs()]
configs += self._get_version_configs()
configs += self._get_repo_configs()
self._configs = util.unique_list(configs)
return self._configs
def _get_packages(self, node_processes):
result = []
result += self.dependencies
result += [(np.package, self.version)
for np in node_processes
if np != SPARK_HISTORY_SERVER]
return g.unique_list(result)
def get_open_ports(self, node_group):
result = []
for service in self.get_services():
for node_process in service.node_processes:
if node_process.ui_name in node_group.node_processes:
result += node_process.open_ports
return util.unique_list(result)
def _install_ssh_keys(self, cluster_context, instances):
slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
masters = cluster_context.filter_instances(instances, SPARK_MASTER)
instances = g.unique_list(masters + slaves)
private_key = cluster_context.cluster.management_private_key
public_key = cluster_context.cluster.management_public_key
g.execute_on_instances(
instances, g.install_ssh_key, 'mapr', private_key, public_key)
g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
def _install_ssh_keys(self, cluster_context, instances):
slaves = cluster_context.filter_instances(instances, SPARK_SLAVE)
masters = cluster_context.filter_instances(instances, SPARK_MASTER)
instances = g.unique_list(masters + slaves)
private_key = cluster_context.cluster.management_private_key
public_key = cluster_context.cluster.management_public_key
g.execute_on_instances(
instances, g.install_ssh_key, 'mapr', private_key, public_key)
g.execute_on_instances(instances, g.authorize_key, 'mapr', public_key)
def _should_restart(self, c_context, instances):
app_services = [
impala.Impala(),
hive.Hive(),
hbase.HBase(),
sqoop.Sqoop2(),
]
instances = [c_context.filter_instances(instances, service=service)
for service in app_services]
return bool(g.unique_list(itertools.chain(*instances)))
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for file in config_files:
util.write_file(instance, file.path, file.data, mode=file.mode,
owner="mapr")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances, write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for path, data in six.iteritems(config_files):
util.mkdir(instance, os.path.dirname(path), owner="root")
util.write_file(instance, path, data, owner="root")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances, write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _write_config_files(self, cluster_context, instances):
LOG.debug('Writing config files')
@el.provision_event()
def write_config_files(instance, config_files):
for file in config_files:
util.write_file(instance,
file.path,
file.data,
mode=file.mode,
owner="mapr")
node_groups = util.unique_list(instances, lambda i: i.node_group)
for node_group in node_groups:
config_files = cluster_context.get_config_files(node_group)
ng_instances = [i for i in node_group.instances if i in instances]
util.execute_on_instances(ng_instances,
write_config_files,
config_files=config_files)
LOG.debug("Config files are successfully written")
def _restart_services(self, cluster_context):
restart = cluster_context.should_be_restarted
for service, instances in six.iteritems(restart):
service.restart(util.unique_list(instances))
def _restart_services(self, cluster_context):
restart = cluster_context.should_be_restarted
for service, instances in six.iteritems(restart):
service.restart(util.unique_list(instances))
def get_node_groups(instances):
return util.unique_list(instances, lambda i: i.node_group)
def get_node_groups(instances):
return util.unique_list(instances, lambda i: i.node_group)
def get_configs(self):
if not self._configs:
configs = [c for s in self.get_services() for c in s.get_configs()]
configs += self._get_version_configs()
self._configs = util.unique_list(configs)
return self._configs