def test_get_hadoop_ssh_keys(self): cluster_dict = { 'name': 'cluster1', 'plugin_name': 'mock_plugin', 'hadoop_version': 'mock_version', 'default_image_id': 'initial', 'node_groups': [tu.make_ng_dict("ng1", "f1", ["s1"], 1)]} cluster1 = conductor.cluster_create(context.ctx(), cluster_dict) (private_key1, public_key1) = c_h.get_hadoop_ssh_keys(cluster1) #should store keys for old cluster cluster1 = conductor.cluster_get(context.ctx(), cluster1) (private_key2, public_key2) = c_h.get_hadoop_ssh_keys(cluster1) self.assertEqual(public_key1, public_key2) self.assertEqual(private_key1, private_key2) #should generate new keys for new cluster cluster_dict.update({'name': 'cluster2'}) cluster2 = conductor.cluster_create(context.ctx(), cluster_dict) (private_key3, public_key3) = c_h.get_hadoop_ssh_keys(cluster2) self.assertNotEqual(public_key1, public_key3) self.assertNotEqual(private_key1, private_key3)
def _extract_configs_to_extra(self, cluster): oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4()) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( cluster, ng, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths(), c_helper.extract_environment_confs(ng.configuration()), append_oozie=( oozie and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" c_helper.get_hadoop_ssh_keys(cluster,extra) return extra
def _extract_configs_to_extra(self, cluster): oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = six.text_type(uuid.uuid4()) for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( cluster, ng, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths(), c_helper.extract_environment_confs(ng.configuration()), append_oozie=(oozie and oozie.node_group.id == ng.id)) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" c_helper.get_hadoop_ssh_keys(cluster, extra) return extra
def _push_configs_to_new_node(self, cluster, extra, instance): ng_extra = extra[instance.node_group.id] private_key, public_key = c_helper.get_hadoop_ssh_keys(cluster) files = { '/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'], '/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'], '/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'], '/tmp/sahara-hadoop-init.sh': ng_extra['setup_script'], 'id_rsa': private_key, 'authorized_keys': public_key } key_cmd = ('sudo mkdir -p /home/hadoop/.ssh/ && ' 'sudo mv id_rsa authorized_keys /home/hadoop/.ssh && ' 'sudo chown -R hadoop:hadoop /home/hadoop/.ssh && ' 'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}') with remote.get_remote(instance) as r: # TODO(aignatov): sudo chown is wrong solution. But it works. r.execute_command( 'sudo chown -R $USER:$USER /etc/hadoop' ) r.execute_command( 'sudo chown -R $USER:$USER /opt/oozie/conf' ) r.write_files_to(files) r.execute_command( 'sudo chmod 0500 /tmp/sahara-hadoop-init.sh' ) r.execute_command( 'sudo /tmp/sahara-hadoop-init.sh ' '>> /tmp/sahara-hadoop-init.log 2>&1') r.execute_command(key_cmd) if c_helper.is_data_locality_enabled(cluster): r.write_file_to( '/etc/hadoop/topology.sh', f.get_file_text( 'plugins/vanilla/v1_2_1/resources/topology.sh')) r.execute_command( 'sudo chmod +x /etc/hadoop/topology.sh' ) self._write_topology_data(r, cluster, extra) self._push_master_configs(r, cluster, extra, instance)
def _push_configs_to_new_node(self, cluster, extra, instance): ng_extra = extra[instance.node_group.id] private_key, public_key = c_helper.get_hadoop_ssh_keys(cluster) files = { "/etc/hadoop/core-site.xml": ng_extra["xml"]["core-site"], "/etc/hadoop/mapred-site.xml": ng_extra["xml"]["mapred-site"], "/etc/hadoop/hdfs-site.xml": ng_extra["xml"]["hdfs-site"], "/tmp/sahara-hadoop-init.sh": ng_extra["setup_script"], "id_rsa": private_key, "authorized_keys": public_key, } key_cmd = ( "sudo mkdir -p /home/hadoop/.ssh/ && " "sudo mv id_rsa authorized_keys /home/hadoop/.ssh && " "sudo chown -R hadoop:hadoop /home/hadoop/.ssh && " "sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}" ) with remote.get_remote(instance) as r: # TODO(aignatov): sudo chown is wrong solution. But it works. r.execute_command("sudo chown -R $USER:$USER /etc/hadoop") r.execute_command("sudo chown -R $USER:$USER /opt/oozie/conf") r.write_files_to(files) r.execute_command("sudo chmod 0500 /tmp/sahara-hadoop-init.sh") r.execute_command("sudo /tmp/sahara-hadoop-init.sh " ">> /tmp/sahara-hadoop-init.log 2>&1") r.execute_command(key_cmd) if c_helper.is_data_locality_enabled(cluster): r.write_file_to( "/etc/hadoop/topology.sh", f.get_file_text("plugins/vanilla/v1_2_1/resources/topology.sh") ) r.execute_command("sudo chmod +x /etc/hadoop/topology.sh") self._write_topology_data(r, cluster, extra) self._push_master_configs(r, cluster, extra, instance)