def scale_cluster(self, cluster, instances): processor = self._get_blueprint_processor(cluster) cluster_spec = clusterspec.ClusterSpec( json.dumps(processor.blueprint), cluster=cluster) rpm = self._get_rpm_uri(cluster_spec) servers = [] for instance in instances: host_role = utils.get_host_role(instance) servers.append(h.HadoopServer(instance, cluster_spec.node_groups [host_role], ambari_rpm=rpm)) ambari_info = self.get_ambari_info(cluster_spec) self._update_ambari_info_credentials(cluster_spec, ambari_info) for server in servers: self._spawn('Ambari provisioning thread', server.provision_ambari, ambari_info) self._wait_for_host_registrations(self._get_num_hosts(cluster), ambari_info) # now add the hosts and the component self._add_hosts_and_components(cluster_spec, servers, ambari_info, cluster.name) self._install_and_start_components(cluster.name, servers, ambari_info)
def create_cluster(self, cluster): version = cluster.hadoop_version handler = self.version_factory.get_version_handler(version) cluster_spec = handler.get_cluster_spec( cluster, self._map_to_user_inputs( version, cluster.cluster_configs)) hosts = self._get_servers(cluster) ambari_info = self.get_ambari_info(cluster_spec) self.cluster_ambari_mapping[cluster.name] = ambari_info rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append( h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) self._provision_cluster( cluster.name, cluster_spec, ambari_info, servers, cluster.hadoop_version) LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec)
def create_cluster(self, cluster): version = cluster.hadoop_version handler = self.version_factory.get_version_handler(version) cluster_spec = handler.get_cluster_spec( cluster, self._map_to_user_inputs( version, cluster.cluster_configs)) hosts = self._get_servers(cluster) ambari_info = self.get_ambari_info(cluster_spec) self.cluster_ambari_mapping[cluster.name] = ambari_info rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append( h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) self._provision_cluster( cluster.name, cluster_spec, ambari_info, servers, cluster.hadoop_version) # add the topology data file and script if rack awareness is # enabled self._configure_topology_for_cluster(cluster, servers) LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec)
def scale_cluster(self, cluster, instances): handler = self.version_factory.get_version_handler( cluster.hadoop_version) ambari_client = handler.get_ambari_client() cluster_spec = handler.get_cluster_spec( cluster, self._map_to_user_inputs( cluster.hadoop_version, cluster.cluster_configs)) rpm = self._get_rpm_uri(cluster_spec) servers = [] for instance in instances: host_role = utils.get_host_role(instance) servers.append(h.HadoopServer(instance, cluster_spec.node_groups [host_role], ambari_rpm=rpm)) ambari_info = self.get_ambari_info(cluster_spec) self._update_ambari_info_credentials(cluster_spec, ambari_info) for server in servers: self._spawn('Ambari provisioning thread', server.provision_ambari, ambari_info) ambari_client.scale_cluster(cluster.name, cluster_spec, servers, self._get_num_hosts(cluster), ambari_info) ambari_client.cleanup(ambari_info)
def create_cluster(self, cluster, cluster_template): if cluster_template is None: raise ValueError("must supply cluster template") cluster_spec = clusterspec.ClusterSpec(cluster_template, cluster=cluster) hosts = self._get_servers(cluster) ambari_host = self._determine_host_for_server_component("AMBARI_SERVER", cluster_spec, hosts) self.cluster_name_to_ambari_host_mapping[cluster.name] = ambari_host rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append(h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) provisioned = self._provision_cluster(cluster.name, cluster_spec, ambari_host, servers) if provisioned: installed = self._install_services(cluster.name, ambari_host) if installed: LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec, hosts) else: raise ex.HadoopProvisionError("Installation of Hadoop stack failed.") else: raise ex.HadoopProvisionError("Provisioning of Hadoop cluster failed.")
def scale_cluster(self, cluster, instances): handler = self.version_factory.get_version_handler( cluster.hadoop_version) ambari_client = handler.get_ambari_client() cluster_spec = handler.get_cluster_spec( cluster, self._map_to_user_inputs( cluster.hadoop_version, cluster.cluster_configs)) rpm = self._get_rpm_uri(cluster_spec) servers = [] for instance in instances: host_role = utils.get_host_role(instance) servers.append(h.HadoopServer(instance, cluster_spec.node_groups [host_role], ambari_rpm=rpm)) ambari_info = self.get_ambari_info(cluster_spec) self._update_ambari_info_credentials(cluster_spec, ambari_info) for server in servers: self._spawn('Ambari provisioning thread', server.provision_ambari, ambari_info, cluster_spec) ambari_client.configure_scaled_cluster_instances( cluster.name, cluster_spec, self._get_num_hosts(cluster), ambari_info) self._configure_topology_for_cluster(cluster, servers) ambari_client.start_scaled_cluster_instances(cluster.name, cluster_spec, servers, ambari_info) ambari_client.cleanup(ambari_info)
def scale_cluster(self, cluster, instances): processor = self._get_blueprint_processor(cluster) cluster_spec = clusterspec.ClusterSpec( json.dumps(processor.blueprint), cluster=cluster) rpm = self._get_rpm_uri(cluster_spec) servers = [] for instance in instances: host_role = utils.get_host_role(instance) servers.append(h.HadoopServer(instance, cluster_spec.node_groups [host_role], ambari_rpm=rpm)) ambari_info = self.get_ambari_info(cluster_spec, self._get_servers(cluster)) for server in servers: self._spawn('Ambari provisioning thread', server.provision_ambari, ambari_info) self._wait_for_host_registrations(self._get_num_hosts(cluster), ambari_info) # now add the hosts and the component self._add_hosts_and_components(cluster_spec, servers, ambari_info, cluster.name) self._install_and_start_components(cluster.name, servers, ambari_info)
def _get_ambari_host(self, servers): # iterate thru servers and find the master server host = next((server for server in servers if utils.get_node_processes(server) is not None and 'AMBARI_SERVER' in utils.get_node_processes(server)), None) if host is None: host = next((server for server in servers if utils.get_host_role(server) == 'MASTER'), None) return host
def _determine_host_for_server_component(self, component, cluster_spec, servers): found_node_group = None node_groups = cluster_spec.node_groups for node_group in node_groups.values(): if component in node_group.components: found_node_group = node_group.name for host in servers: if utils.get_host_role(host) == found_node_group: return host raise Exception("Server component [{0}] not specified in configuration".format(component))
def _get_ambari_host(self, servers): # iterate thru servers and find the master server host = next( ( server for server in servers if utils.get_node_processes(server) is not None and "AMBARI_SERVER" in utils.get_node_processes(server) ), None, ) if host is None: host = next((server for server in servers if utils.get_host_role(server) == "MASTER"), None) return host
def _determine_host_for_server_component(self, component, cluster_spec, servers): found_node_group = None node_groups = cluster_spec.node_groups for node_group in node_groups.values(): if component in node_group.components: found_node_group = node_group.name for host in servers: if utils.get_host_role(host) == found_node_group: return host raise Exception( 'Server component [{0}] not specified in configuration'.format( component))
def _generate_host_manifest(self): host_manifest = {} hosts = [] host_id = 1 for server in self.servers: instance_info = n_helper.get_instance_info(server) hosts.append({'host_id': host_id, 'hostname': server.hostname, 'role': utils.get_host_role(server), 'vm_image': instance_info.image, 'vm_flavor': instance_info.flavor, 'public_ip': server.management_ip, 'private_ip': server.internal_ip}) host_id += 1 host_manifest['hosts'] = hosts return json.dumps(host_manifest).strip('{}')
def _generate_host_manifest(self, servers): host_manifest = {} hosts = [] host_id = 1 for server in servers: instance_info = n_helper.get_instance_info(server) hosts.append({ 'host_id': host_id, 'hostname': server.hostname, 'role': utils.get_host_role(server), 'vm_image': instance_info.image, 'vm_flavor': instance_info.flavor, 'public_ip': server.management_ip, 'private_ip': server.internal_ip }) host_id += 1 host_manifest['hosts'] = hosts return json.dumps(host_manifest).strip('{}')
def _generate_host_manifest(self, servers): host_manifest = {} hosts = [] host_id = 1 for server in servers: instance_info = n_helper.get_instance_info(server) hosts.append( { "host_id": host_id, "hostname": server.hostname, "role": utils.get_host_role(server), "vm_image": instance_info.image, "vm_flavor": instance_info.flavor, "public_ip": server.management_ip, "private_ip": server.internal_ip, } ) host_id += 1 host_manifest["hosts"] = hosts return json.dumps(host_manifest).strip("{}")
def create_cluster(self, cluster, cluster_template): if cluster_template is None: raise ValueError('must supply cluster template') cluster_spec = clusterspec.ClusterSpec(cluster_template, cluster=cluster) hosts = self._get_servers(cluster) ambari_host = self._determine_host_for_server_component( 'AMBARI_SERVER', cluster_spec, hosts) self.cluster_name_to_ambari_host_mapping[cluster.name] = ambari_host rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append( h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) provisioned = self._provision_cluster(cluster.name, cluster_spec, ambari_host, servers) if provisioned: installed = self._install_services(cluster.name, ambari_host) if installed: LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec, hosts) else: raise ex.HadoopProvisionError( 'Installation of Hadoop stack failed.') else: raise ex.HadoopProvisionError( 'Provisioning of Hadoop cluster failed.')
def create_cluster(self, cluster, cluster_template): if cluster_template is None: raise ValueError('must supply cluster template') cluster_spec = clusterspec.ClusterSpec( cluster_template, cluster=cluster) hosts = self._get_servers(cluster) ambari_info = self.get_ambari_info(cluster_spec) self.cluster_ambari_mapping[cluster.name] = ambari_info rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append( h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) provisioned = self._provision_cluster( cluster.name, cluster_spec, ambari_info, servers) if provisioned: installed = self._install_services(cluster.name, ambari_info) if installed: LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec, ambari_info) else: raise ex.HadoopProvisionError( 'Installation of Hadoop stack failed.') else: raise ex.HadoopProvisionError( 'Provisioning of Hadoop cluster failed.')