def decommission_dn(nn, inst_to_be_deleted, survived_inst): with remote.get_remote(nn) as r: r.write_file_to('/etc/hadoop/dn.excl', utils.generate_fqdn_host_names(inst_to_be_deleted)) run.refresh_nodes(remote.get_remote(nn), "dfsadmin") context.sleep(3) att_amount = 10 while att_amount: cmd = r.execute_command( "sudo su -c 'hadoop dfsadmin -report' hadoop") all_found = True datanodes_info = parse_dfs_report(cmd[1]) for i in inst_to_be_deleted: for dn in datanodes_info: if (dn["Name"].startswith(i.internal_ip)) and ( dn["Decommission Status"] != "Decommissioned"): all_found = False break if all_found: r.write_files_to({ '/etc/hadoop/dn.incl': utils.generate_fqdn_host_names(survived_inst), '/etc/hadoop/dn.excl': "", }) break context.sleep(3) att_amount -= 1 if not att_amount: raise Exception("Cannot finish decommission")
def decommission_dn(nn, inst_to_be_deleted, survived_inst): with remote.get_remote(nn) as r: r.write_file_to("/etc/hadoop/dn.excl", utils.generate_fqdn_host_names(inst_to_be_deleted)) run.refresh_nodes(remote.get_remote(nn), "dfsadmin") context.sleep(3) att_amount = 100 while att_amount: cmd = r.execute_command("sudo su -c 'hadoop dfsadmin -report' hadoop") all_found = True datanodes_info = parse_dfs_report(cmd[1]) for i in inst_to_be_deleted: for dn in datanodes_info: if (dn["Name"].startswith(i.internal_ip)) and (dn["Decommission Status"] != "Decommissioned"): all_found = False break if all_found: r.write_files_to( {"/etc/hadoop/dn.incl": utils.generate_fqdn_host_names(survived_inst), "/etc/hadoop/dn.excl": ""} ) break context.sleep(3) att_amount -= 1 if not att_amount: raise Exception("Cannot finish decommission")
def decommission_tt(jt, inst_to_be_deleted, survived_inst): with remote.get_remote(jt) as r: r.write_file_to("/etc/hadoop/tt.excl", utils.generate_fqdn_host_names(inst_to_be_deleted)) run.refresh_nodes(remote.get_remote(jt), "mradmin") context.sleep(3) r.write_files_to( {"/etc/hadoop/tt.incl": utils.generate_fqdn_host_names(survived_inst), "/etc/hadoop/tt.excl": ""} )
def decommission_tt(jt, inst_to_be_deleted, survived_inst): with remote.get_remote(jt) as r: r.write_file_to('/etc/hadoop/tt.excl', utils.generate_fqdn_host_names(inst_to_be_deleted)) run.refresh_nodes(remote.get_remote(jt), "mradmin") context.sleep(3) r.write_files_to({ '/etc/hadoop/tt.incl': utils.generate_fqdn_host_names(survived_inst), '/etc/hadoop/tt.excl': "", })
def _push_configs_to_nodes(self, cluster, instances=None): extra = self._extract_configs_to_extra(cluster) if instances is None: instances = utils.get_instances(cluster) for inst in instances: ng_extra = extra[inst.node_group.id] files = { '/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'], '/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'], '/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'], '/tmp/savanna-hadoop-init.sh': ng_extra['setup_script'] } with remote.get_remote(inst) as r: # TODO(aignatov): sudo chown is wrong solution. But it works. r.execute_command('sudo chown -R $USER:$USER /etc/hadoop') r.execute_command('sudo chown -R $USER:$USER /opt/oozie/conf') r.write_files_to(files) r.execute_command( 'sudo chmod 0500 /tmp/savanna-hadoop-init.sh') r.execute_command('sudo /tmp/savanna-hadoop-init.sh ' '>> /tmp/savanna-hadoop-init.log 2>&1') nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) with remote.get_remote(nn) as r: r.write_file_to( '/etc/hadoop/dn.incl', utils.generate_fqdn_host_names(utils.get_datanodes(cluster))) if jt: with remote.get_remote(jt) as r: r.write_file_to( '/etc/hadoop/tt.incl', utils.generate_fqdn_host_names( utils.get_tasktrackers(cluster))) oozie = utils.get_oozie(cluster) if oozie: with remote.get_remote(oozie) as r: r.write_file_to( '/opt/oozie/conf/oozie-site.xml', extra[oozie.node_group.id]['xml']['oozie-site'])
def _push_jobtracker_configs(self, cluster, r): r.write_file_to( '/etc/hadoop/tt.incl', utils.generate_fqdn_host_names(utils.get_tasktrackers(cluster)))
def _push_namenode_configs(self, cluster, r): r.write_file_to( '/etc/hadoop/dn.incl', utils.generate_fqdn_host_names(utils.get_datanodes(cluster)))
def _push_jobtracker_configs(self, cluster, r): r.write_file_to('/etc/hadoop/tt.incl', utils.generate_fqdn_host_names( utils.get_tasktrackers(cluster)))
def _push_namenode_configs(self, cluster, r): r.write_file_to('/etc/hadoop/dn.incl', utils.generate_fqdn_host_names( utils.get_datanodes(cluster)))