def decommission_dn(nn, inst_to_be_deleted, survived_inst):
with remote.get_remote(nn) as r:
r.write_file_to('/etc/hadoop/dn.excl',
utils.generate_fqdn_host_names(inst_to_be_deleted))
run.refresh_nodes(remote.get_remote(nn), "dfsadmin")
context.sleep(3)
att_amount = 10
while att_amount:
cmd = r.execute_command(
"sudo su -c 'hadoop dfsadmin -report' hadoop")
all_found = True
datanodes_info = parse_dfs_report(cmd[1])
for i in inst_to_be_deleted:
for dn in datanodes_info:
if (dn["Name"].startswith(i.internal_ip)) and (
dn["Decommission Status"] != "Decommissioned"):
all_found = False
break
if all_found:
r.write_files_to({
'/etc/hadoop/dn.incl':
utils.generate_fqdn_host_names(survived_inst),
'/etc/hadoop/dn.excl':
"",
})
break
context.sleep(3)
att_amount -= 1
if not att_amount:
raise Exception("Cannot finish decommission")
def start_cluster(self, cluster):
nn_instance = utils.get_namenode(cluster)
datanodes = utils.get_datanodes(cluster)
jt_instance = utils.get_jobtracker(cluster)
tasktrackers = utils.get_tasktrackers(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_process(r, "namenode")
snns = utils.get_secondarynamenodes(cluster)
if snns:
for snn in snns:
run.start_process(remote.get_remote(snn), "secondarynamenode")
for dn in datanodes:
run.start_process(remote.get_remote(dn), "datanode")
LOG.info("HDFS service at '%s' has been started",
nn_instance.hostname)
if jt_instance:
run.start_process(remote.get_remote(jt_instance), "jobtracker")
for tt in tasktrackers:
run.start_process(remote.get_remote(tt), "tasktracker")
LOG.info("MapReduce service at '%s' has been started",
jt_instance.hostname)
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
def decommission_dn(nn, inst_to_be_deleted, survived_inst):
with remote.get_remote(nn) as r:
r.write_file_to("/etc/hadoop/dn.excl", utils.generate_fqdn_host_names(inst_to_be_deleted))
run.refresh_nodes(remote.get_remote(nn), "dfsadmin")
context.sleep(3)
att_amount = 100
while att_amount:
cmd = r.execute_command("sudo su -c 'hadoop dfsadmin -report' hadoop")
all_found = True
datanodes_info = parse_dfs_report(cmd[1])
for i in inst_to_be_deleted:
for dn in datanodes_info:
if (dn["Name"].startswith(i.internal_ip)) and (dn["Decommission Status"] != "Decommissioned"):
all_found = False
break
if all_found:
r.write_files_to(
{"/etc/hadoop/dn.incl": utils.generate_fqdn_host_names(survived_inst), "/etc/hadoop/dn.excl": ""}
)
break
context.sleep(3)
att_amount -= 1
if not att_amount:
raise Exception("Cannot finish decommission")
def decommission_tt(jt, inst_to_be_deleted, survived_inst):
with remote.get_remote(jt) as r:
r.write_file_to("/etc/hadoop/tt.excl", utils.generate_fqdn_host_names(inst_to_be_deleted))
run.refresh_nodes(remote.get_remote(jt), "mradmin")
context.sleep(3)
r.write_files_to(
{"/etc/hadoop/tt.incl": utils.generate_fqdn_host_names(survived_inst), "/etc/hadoop/tt.excl": ""}
)
def scale_cluster(self, cluster, instances):
self._setup_instances(cluster, instances)
run.refresh_nodes(remote.get_remote(utils.get_namenode(cluster)),
"dfsadmin")
jt = utils.get_jobtracker(cluster)
if jt:
run.refresh_nodes(remote.get_remote(jt), "mradmin")
self._start_tt_dn_processes(instances)
def scale_cluster(self, cluster, instances):
self._setup_instances(cluster, instances)
run.refresh_nodes(remote.get_remote(
utils.get_namenode(cluster)), "dfsadmin")
jt = utils.get_jobtracker(cluster)
if jt:
run.refresh_nodes(remote.get_remote(jt), "mradmin")
self._start_tt_dn_processes(instances)
def decommission_tt(jt, inst_to_be_deleted, survived_inst):
with remote.get_remote(jt) as r:
r.write_file_to('/etc/hadoop/tt.excl',
utils.generate_fqdn_host_names(inst_to_be_deleted))
run.refresh_nodes(remote.get_remote(jt), "mradmin")
context.sleep(3)
r.write_files_to({
'/etc/hadoop/tt.incl':
utils.generate_fqdn_host_names(survived_inst),
'/etc/hadoop/tt.excl':
"",
})
def _push_configs_to_nodes(self, cluster, instances=None):
extra = self._extract_configs_to_extra(cluster)
if instances is None:
instances = utils.get_instances(cluster)
for inst in instances:
ng_extra = extra[inst.node_group.id]
files = {
'/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'],
'/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'],
'/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'],
'/tmp/savanna-hadoop-init.sh': ng_extra['setup_script']
}
with remote.get_remote(inst) as r:
# TODO(aignatov): sudo chown is wrong solution. But it works.
r.execute_command(
'sudo chown -R $USER:$USER /etc/hadoop'
)
r.execute_command(
'sudo chown -R $USER:$USER /opt/oozie/conf'
)
r.write_files_to(files)
r.execute_command(
'sudo chmod 0500 /tmp/savanna-hadoop-init.sh'
)
r.execute_command(
'sudo /tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
with remote.get_remote(nn) as r:
r.write_file_to('/etc/hadoop/dn.incl', utils.
generate_fqdn_host_names(
utils.get_datanodes(cluster)))
if jt:
with remote.get_remote(jt) as r:
r.write_file_to('/etc/hadoop/tt.incl', utils.
generate_fqdn_host_names(
utils.get_tasktrackers(cluster)))
oozie = utils.get_oozie(cluster)
if oozie:
with remote.get_remote(oozie) as r:
r.write_file_to('/opt/oozie/conf/oozie-site.xml',
extra[oozie.node_group.id]
['xml']['oozie-site'])
def _push_configs_to_nodes(self, cluster, instances=None):
extra = self._extract_configs_to_extra(cluster)
if instances is None:
instances = utils.get_instances(cluster)
for inst in instances:
ng_extra = extra[inst.node_group.id]
files = {
'/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'],
'/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'],
'/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'],
'/tmp/savanna-hadoop-init.sh': ng_extra['setup_script']
}
with remote.get_remote(inst) as r:
# TODO(aignatov): sudo chown is wrong solution. But it works.
r.execute_command(
'sudo chown -R $USER:$USER /etc/hadoop'
)
r.execute_command(
'sudo chown -R $USER:$USER /opt/oozie/conf'
)
r.write_files_to(files)
r.execute_command(
'sudo chmod 0500 /tmp/savanna-hadoop-init.sh'
)
r.execute_command(
'sudo /tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
with remote.get_remote(nn) as r:
r.write_file_to('/etc/hadoop/dn.incl', utils.
generate_fqdn_host_names(
utils.get_datanodes(cluster)))
if jt:
with remote.get_remote(jt) as r:
r.write_file_to('/etc/hadoop/tt.incl', utils.
generate_fqdn_host_names(
utils.get_tasktrackers(cluster)))
oozie = utils.get_oozie(cluster)
if oozie:
with remote.get_remote(oozie) as r:
r.write_file_to('/opt/oozie/conf/oozie-site.xml',
extra[oozie.node_group.id]
['xml']['oozie-site'])
def scale_cluster(self, cluster, instances):
self._push_configs_to_nodes(cluster, instances=instances)
self._write_hadoop_user_keys(cluster.private_key, instances)
run.refresh_nodes(remote.get_remote(utils.get_namenode(cluster)),
"dfsadmin")
jt = utils.get_jobtracker(cluster)
if jt:
run.refresh_nodes(remote.get_remote(jt), "mradmin")
for i in instances:
with remote.get_remote(i) as r:
if "datanode" in i.node_group.node_processes:
run.start_process(r, "datanode")
if "tasktracker" in i.node_group.node_processes:
run.start_process(r, "tasktracker")
def _configure_instances(cluster):
"""Configure active instances.
* generate /etc/hosts
* setup passwordless login
* etc.
"""
hosts = _generate_etc_hosts(cluster)
for node_group in cluster.node_groups:
for instance in node_group.instances:
with remote.get_remote(instance) as r:
r.write_file_to('etc-hosts', hosts)
r.execute_command('sudo mv etc-hosts /etc/hosts')
# wait generate id_rsa key
timeout = 10
cur_time = 0
while cur_time < timeout:
code, _ = r.execute_command('ls .ssh/id_rsa',
raise_when_error=False)
if code:
cur_time += 1
context.sleep(1)
else:
break
else:
raise RuntimeError("Error getting user private key")
r.execute_command('sudo chown $USER:$USER .ssh/id_rsa')
r.execute_command('chmod 400 .ssh/id_rsa')
def _check_if_up(instance):
if hasattr(instance, '_is_up'):
return True
server = nova.get_instance_info(instance)
if server.status == 'ERROR':
# TODO(slukjanov): replace with specific error
raise RuntimeError("node %s has error status" % server.name)
if server.status != 'ACTIVE':
return False
if len(server.networks) == 0:
return False
if not networks.init_instances_ips(instance, server):
return False
try:
exit_code, _ = remote.get_remote(instance).execute_command("hostname")
if exit_code:
return False
except Exception as ex:
LOG.debug("Can't login to node %s (%s), reason %s",
server.name, instance.management_ip, ex)
return False
instance._is_up = True
return True
def scale_cluster(self, cluster, instances):
self._push_configs_to_nodes(cluster, instances=instances)
self._write_hadoop_user_keys(cluster.private_key,
instances)
run.refresh_nodes(remote.get_remote(
utils.get_namenode(cluster)), "dfsadmin")
jt = utils.get_jobtracker(cluster)
if jt:
run.refresh_nodes(remote.get_remote(jt), "mradmin")
for i in instances:
with remote.get_remote(i) as r:
if "datanode" in i.node_group.node_processes:
run.start_process(r, "datanode")
if "tasktracker" in i.node_group.node_processes:
run.start_process(r, "tasktracker")
def create_workflow_dir(where, job, hdfs_user):
constructed_dir = '/user/hadoop/'
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid())
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir, hdfs_user)
return constructed_dir
def create_workflow_dir(where, job, hdfs_user):
constructed_dir = '/user/%s/' % hdfs_user
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, six.text_type(uuid.uuid4()))
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir, hdfs_user)
return constructed_dir
def create_workflow_dir(where, job, hdfs_user):
constructed_dir = '/user/%s/' % hdfs_user
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, six.text_type(uuid.uuid4()))
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir, hdfs_user)
return constructed_dir
def create_workflow_dir(where, job):
constructed_dir = '/user/hadoop/'
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid())
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir)
return constructed_dir
def upload_job_file(where, job_dir, job_origin, job):
main_binary = conductor.job_binary_get_raw_data(context.ctx(),
job_origin.url)
if job.type == 'Jar':
job_dir += '/lib'
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, main_binary, main_res_names[job.type], job_dir)
return "%s/%s" % (job_dir, main_res_names[job.type])
def upload_job_file(where, job_dir, job_origin, job):
main_binary = conductor.job_binary_get_raw_data(context.ctx(),
job_origin.url)
if job.type == 'Jar':
job_dir += '/lib'
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, main_binary, main_res_names[job.type], job_dir)
return "%s/%s" % (job_dir, main_res_names[job.type])
def start_cluster(self, cluster):
nn_instance = utils.get_namenode(cluster)
datanodes = utils.get_datanodes(cluster)
jt_instance = utils.get_jobtracker(cluster)
tasktrackers = utils.get_tasktrackers(cluster)
oozie = utils.get_oozie(cluster)
hive_server = utils.get_hiveserver(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_process(r, "namenode")
snns = utils.get_secondarynamenodes(cluster)
if snns:
for snn in snns:
run.start_process(remote.get_remote(snn), "secondarynamenode")
for dn in datanodes:
run.start_process(remote.get_remote(dn), "datanode")
LOG.info("HDFS service at '%s' has been started",
nn_instance.hostname)
if jt_instance:
run.start_process(remote.get_remote(jt_instance), "jobtracker")
for tt in tasktrackers:
run.start_process(remote.get_remote(tt), "tasktracker")
LOG.info("MapReduce service at '%s' has been started",
jt_instance.hostname)
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname)
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
nn_instance.hostname)
if hive_server:
with remote.get_remote(nn_instance) as r:
run.hive_create_warehouse_dir(r)
if c_helper.is_mysql_enable(cluster):
with remote.get_remote(hive_server) as h:
if not oozie or hive_server.hostname != oozie.hostname:
run.mysql_start(h, hive_server)
run.hive_create_db(h)
run.hive_metastore_start(h)
LOG.info("Hive Metastore server at %s has been started",
hive_server.hostname)
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
nn_instance = utils.get_namenode(cluster)
datanodes = utils.get_datanodes(cluster)
jt_instance = utils.get_jobtracker(cluster)
tasktrackers = utils.get_tasktrackers(cluster)
oozie = utils.get_oozie(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_process(r, "namenode")
snns = utils.get_secondarynamenodes(cluster)
if snns:
for snn in snns:
run.start_process(remote.get_remote(snn), "secondarynamenode")
for dn in datanodes:
run.start_process(remote.get_remote(dn), "datanode")
LOG.info("HDFS service at '%s' has been started", nn_instance.hostname)
if jt_instance:
run.start_process(remote.get_remote(jt_instance), "jobtracker")
for tt in tasktrackers:
run.start_process(remote.get_remote(tt), "tasktracker")
LOG.info("MapReduce service at '%s' has been started",
jt_instance.hostname)
if oozie:
with remote.get_remote(oozie) as r:
run.oozie_share_lib(r, nn_instance.hostname)
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
nn_instance.hostname)
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
def _push_configs_to_nodes(self, cluster, instances=None):
if instances is None:
instances = utils.get_instances(cluster)
for inst in instances:
files = {
'/etc/hadoop/core-site.xml': inst.node_group.extra['xml'][
'core-site'],
'/etc/hadoop/mapred-site.xml': inst.node_group.extra['xml'][
'mapred-site'],
'/etc/hadoop/hdfs-site.xml': inst.node_group.extra['xml'][
'hdfs-site'],
'/tmp/savanna-hadoop-init.sh': inst.node_group.extra[
'setup_script']
}
with remote.get_remote(inst) as r:
r.execute_command(
'sudo chown -R $USER:$USER /etc/hadoop'
)
r.write_files_to(files)
r.execute_command(
'sudo chmod 0500 /tmp/savanna-hadoop-init.sh'
)
r.execute_command(
'sudo /tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
with remote.get_remote(nn) as r:
r.write_file_to('/etc/hadoop/dn.incl', utils.
generate_fqdn_host_names(
utils.get_datanodes(cluster)))
if jt:
with remote.get_remote(jt) as r:
r.write_file_to('/etc/hadoop/tt.incl', utils.
generate_fqdn_host_names(
utils.get_tasktrackers(cluster)))
def _push_configs_to_existing_node(self, cluster, extra, instance):
node_processes = instance.node_group.node_processes
need_update = (c_helper.is_data_locality_enabled(cluster)
or 'namenode' in node_processes
or 'jobtracker' in node_processes
or 'oozie' in node_processes
or 'hiveserver' in node_processes)
if not need_update:
return
with remote.get_remote(instance) as r:
self._write_topology_data(r, cluster, extra)
self._push_master_configs(r, cluster, extra, instance)
def _mount_volume(instance, device_path, mount_point):
codes = []
with remote.get_remote(instance) as r:
code, _ = r.execute_command('sudo mkdir -p %s' % mount_point)
codes.append(code)
code, _ = r.execute_command('sudo mkfs.ext4 %s' % device_path)
codes.append(code)
code, _ = r.execute_command('sudo mount %s %s' % (device_path,
mount_point))
codes.append(code)
if any(codes):
raise RuntimeError("Error mounting volume to instance %s" %
instance.instance_id)
def _push_configs_to_existing_node(self, cluster, extra, instance):
node_processes = instance.node_group.node_processes
need_update = (c_helper.is_data_locality_enabled(cluster) or
'namenode' in node_processes or
'jobtracker' in node_processes or
'oozie' in node_processes or
'hiveserver' in node_processes)
if not need_update:
return
with remote.get_remote(instance) as r:
self._write_topology_data(r, cluster, extra)
self._push_master_configs(r, cluster, extra, instance)
def _write_hadoop_user_keys(self, private_key, instances):
public_key = crypto.private_key_to_public_key(private_key)
files = {'id_rsa': private_key, 'authorized_keys': public_key}
mv_cmd = 'sudo mkdir -p /home/hadoop/.ssh/; ' \
'sudo mv id_rsa authorized_keys /home/hadoop/.ssh ; ' \
'sudo chown -R hadoop:hadoop /home/hadoop/.ssh; ' \
'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}'
for instance in instances:
with remote.get_remote(instance) as r:
r.write_files_to(files)
r.execute_command(mv_cmd)
def _write_hadoop_user_keys(self, instances, private_key, public_key):
files = {
'id_rsa': private_key,
'authorized_keys': public_key
}
mv_cmd = 'sudo mkdir -p /home/hadoop/.ssh/; ' \
'sudo mv id_rsa authorized_keys /home/hadoop/.ssh ; ' \
'sudo chown -R hadoop:hadoop /home/hadoop/.ssh; ' \
'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}'
for instance in instances:
with remote.get_remote(instance) as r:
r.write_files_to(files)
r.execute_command(mv_cmd)
def upload_job_files(where, job_dir, job_origin):
mains = job_origin.mains or []
libs = job_origin.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib")
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def _get_device_paths(instance):
code, part_info = remote.get_remote(instance).execute_command(
'cat /proc/partitions')
if code:
raise RuntimeError("Unable get device paths info")
out = part_info.split('\n')[1:]
device_paths = []
for line in out:
spl = line.split()
if len(spl) > 3:
dev = spl[3]
if not re.search('\d$', dev):
device_paths.append('/dev/' + dev)
return device_paths
def _push_configs_to_new_node(self, cluster, extra, instance):
ng_extra = extra[instance.node_group.id]
private_key, public_key = c_helper.get_hadoop_ssh_keys(cluster)
files = {
'/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'],
'/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'],
'/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'],
'/tmp/savanna-hadoop-init.sh': ng_extra['setup_script'],
'id_rsa': private_key,
'authorized_keys': public_key
}
key_cmd = 'sudo mkdir -p /home/hadoop/.ssh/ && ' \
'sudo mv id_rsa authorized_keys /home/hadoop/.ssh && ' \
'sudo chown -R hadoop:hadoop /home/hadoop/.ssh && ' \
'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}'
with remote.get_remote(instance) as r:
# TODO(aignatov): sudo chown is wrong solution. But it works.
r.execute_command(
'sudo chown -R $USER:$USER /etc/hadoop'
)
r.execute_command(
'sudo chown -R $USER:$USER /opt/oozie/conf'
)
r.write_files_to(files)
r.execute_command(
'sudo chmod 0500 /tmp/savanna-hadoop-init.sh'
)
r.execute_command(
'sudo /tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
r.execute_command(key_cmd)
if c_helper.is_data_locality_enabled(cluster):
r.write_file_to(
'/etc/hadoop/topology.sh',
f.get_file_text(
'plugins/vanilla/resources/topology.sh'))
r.execute_command(
'sudo chmod +x /etc/hadoop/topology.sh'
)
self._write_topology_data(r, cluster, extra)
self._push_master_configs(r, cluster, extra, instance)
def upload_job_files(where, job_dir, job, hdfs_user):
mains = job.mains or []
libs = job.libs or []
uploaded_paths = []
with remote.get_remote(where) as r:
for main in mains:
raw_data = dispatch.get_raw_binary(main)
h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user)
uploaded_paths.append(job_dir + '/' + main.name)
for lib in libs:
raw_data = dispatch.get_raw_binary(lib)
h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib",
hdfs_user)
uploaded_paths.append(job_dir + '/lib/' + lib.name)
return uploaded_paths
def _await_datanodes(self, cluster):
datanodes_count = len(utils.get_datanodes(cluster))
if datanodes_count < 1:
return
LOG.info("Waiting %s datanodes to start up" % datanodes_count)
with remote.get_remote(utils.get_namenode(cluster)) as r:
while True:
if run.check_datanodes_count(r, datanodes_count):
LOG.info('Datanodes on cluster %s has been started' %
cluster.name)
return
context.sleep(1)
if not g.check_cluster_exists(cluster):
LOG.info(
'Stop waiting datanodes on cluster %s since it has '
'been deleted' % cluster.name)
return
def _await_datanodes(self, cluster):
datanodes_count = len(utils.get_datanodes(cluster))
if datanodes_count < 1:
return
LOG.info("Waiting %s datanodes to start up" % datanodes_count)
with remote.get_remote(utils.get_namenode(cluster)) as r:
while True:
if run.check_datanodes_count(r, datanodes_count):
LOG.info(
'Datanodes on cluster %s has been started' %
cluster.name)
return
context.sleep(1)
if not g.check_cluster_exists(cluster):
LOG.info(
'Stop waiting datanodes on cluster %s since it has '
'been deleted' % cluster.name)
return
def start_cluster(self, cluster):
nn_instance = utils.get_namenode(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_processes(r, "namenode")
for snn in utils.get_secondarynamenodes(cluster):
run.start_processes(remote.get_remote(snn), "secondarynamenode")
jt_instance = utils.get_jobtracker(cluster)
if jt_instance:
run.start_processes(remote.get_remote(jt_instance), "jobtracker")
self._start_tt_dn_processes(utils.get_instances(cluster))
self._await_datanodes(cluster)
LOG.info("Hadoop services in cluster %s have been started" %
cluster.name)
oozie = utils.get_oozie(cluster)
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname())
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
nn_instance.hostname())
hive_server = utils.get_hiveserver(cluster)
if hive_server:
with remote.get_remote(nn_instance) as r:
run.hive_create_warehouse_dir(r)
if c_helper.is_mysql_enable(cluster):
with remote.get_remote(hive_server) as h:
if not oozie or hive_server.hostname() != oozie.hostname():
run.mysql_start(h, hive_server)
run.hive_create_db(h)
run.hive_metastore_start(h)
LOG.info("Hive Metastore server at %s has been started",
hive_server.hostname())
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
instances = utils.get_instances(cluster)
nn_instance = utils.get_namenode(cluster)
jt_instance = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive_server = utils.get_hiveserver(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_processes(r, "namenode")
for snn in utils.get_secondarynamenodes(cluster):
run.start_processes(remote.get_remote(snn), "secondarynamenode")
if jt_instance:
run.start_processes(remote.get_remote(jt_instance), "jobtracker")
self._start_tt_dn_processes(instances)
LOG.info("Hadoop services in cluster %s have been started" %
cluster.name)
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname)
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
nn_instance.hostname)
if hive_server:
with remote.get_remote(nn_instance) as r:
run.hive_create_warehouse_dir(r)
if c_helper.is_mysql_enable(cluster):
with remote.get_remote(hive_server) as h:
if not oozie or hive_server.hostname != oozie.hostname:
run.mysql_start(h, hive_server)
run.hive_create_db(h)
run.hive_metastore_start(h)
LOG.info("Hive Metastore server at %s has been started",
hive_server.hostname)
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
def _push_configs_to_new_node(self, cluster, extra, instance):
ng_extra = extra[instance.node_group.id]
private_key, public_key = c_helper.get_hadoop_ssh_keys(cluster)
files = {
'/etc/hadoop/core-site.xml': ng_extra['xml']['core-site'],
'/etc/hadoop/mapred-site.xml': ng_extra['xml']['mapred-site'],
'/etc/hadoop/hdfs-site.xml': ng_extra['xml']['hdfs-site'],
'/tmp/savanna-hadoop-init.sh': ng_extra['setup_script'],
'id_rsa': private_key,
'authorized_keys': public_key
}
key_cmd = 'sudo mkdir -p /home/hadoop/.ssh/ && ' \
'sudo mv id_rsa authorized_keys /home/hadoop/.ssh && ' \
'sudo chown -R hadoop:hadoop /home/hadoop/.ssh && ' \
'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}'
with remote.get_remote(instance) as r:
# TODO(aignatov): sudo chown is wrong solution. But it works.
r.execute_command('sudo chown -R $USER:$USER /etc/hadoop')
r.execute_command('sudo chown -R $USER:$USER /opt/oozie/conf')
r.write_files_to(files)
r.execute_command('sudo chmod 0500 /tmp/savanna-hadoop-init.sh')
r.execute_command('sudo /tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
r.execute_command(key_cmd)
if c_helper.is_data_locality_enabled(cluster):
r.write_file_to(
'/etc/hadoop/topology.sh',
f.get_file_text(
'plugins/vanilla/v1_2_1/resources/topology.sh'))
r.execute_command('sudo chmod +x /etc/hadoop/topology.sh')
self._write_topology_data(r, cluster, extra)
self._push_master_configs(r, cluster, extra, instance)
def upload_workflow_file(where, job_dir, wf_xml, hdfs_user):
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir, hdfs_user)
return "%s/workflow.xml" % job_dir
def configure_workflow_if_needed(self, cluster, wf_dir):
h_s = u.get_hiveserver(cluster)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
hdfs_user = plugin.get_hdfs_user()
h.copy_from_local(remote.get_remote(h_s),
plugin.get_hive_config_path(), wf_dir, hdfs_user)
def _connect_to_vm(self):
LOG.info('Connecting to VM: {0}'.format(self.instance.management_ip))
return remote.get_remote(self.instance).ssh_connection()
def __init__(self, instance, node_group):
self.instance = instance
self.node_group = node_group
self._remote = remote.get_remote(instance)
def remote(self):
return remote.get_remote(self)
def _connect_to_vm(self):
LOG.info(
'Connecting to VM: {0}'.format(self.instance.management_ip))
return remote.get_remote(self.instance).ssh_connection()
def __init__(self, instance, node_group, ambari_rpm=None):
self.instance = instance
self.node_group = node_group
self.ambari_rpm = ambari_rpm or AMBARI_RPM
self._remote = remote.get_remote(instance)
def configure_workflow_if_needed(self, cluster, wf_dir):
h_s = u.get_hiveserver(cluster)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
h.copy_from_local(remote.get_remote(h_s),
plugin.get_hive_config_path(), wf_dir)
def upload_hive_site(cluster, wf_dir):
h_s = u.get_hiveserver(cluster)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
h.copy_from_local(remote.get_remote(h_s),
plugin.get_hive_config_path(), wf_dir)
def upload_workflow_file(where, job_dir, wf_xml):
with remote.get_remote(where) as r:
h.put_file_to_hdfs(r, wf_xml, "workflow.xml", job_dir)
return "%s/workflow.xml" % job_dir
def remote(self):
return remote.get_remote(self)
