def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozies_hostnames = [o.hostname for o in utils.get_oozies(cluster)]
extra = dict()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname
if jt else None,
oozies_hostnames),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration)
)
}
return extra
def _set_cluster_info(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
info = cluster.info
if jt and jt.management_ip:
info['MapReduce'] = {
'Web UI': 'http://%s:50030' % jt.management_ip
}
if nn and nn.management_ip:
info['HDFS'] = {
'Web UI': 'http://%s:50070' % nn.management_ip
}
info['JobFlow'] = {}
for oozie in utils.get_oozies(cluster):
if oozie.management_ip:
info['JobFlow'].update({
'Oozie: %s' % oozie.hostname:
'http://%s:11000' % oozie.management_ip
})
def start_cluster(self, cluster):
nn_instance = utils.get_namenode(cluster)
datanodes = utils.get_datanodes(cluster)
jt_instance = utils.get_jobtracker(cluster)
tasktrackers = utils.get_tasktrackers(cluster)
oozies = utils.get_oozies(cluster)
with remote.get_remote(nn_instance) as r:
run.format_namenode(r)
run.start_process(r, "namenode")
snns = utils.get_secondarynamenodes(cluster)
if snns:
for snn in snns:
run.start_process(remote.get_remote(snn), "secondarynamenode")
for dn in datanodes:
run.start_process(remote.get_remote(dn), "datanode")
LOG.info("HDFS service at '%s' has been started",
nn_instance.hostname)
if jt_instance:
run.start_process(remote.get_remote(jt_instance), "jobtracker")
for tt in tasktrackers:
run.start_process(remote.get_remote(tt), "tasktracker")
LOG.info("MapReduce service at '%s' has been started",
jt_instance.hostname)
for oozie in oozies:
with remote.get_remote(oozie) as r:
run.oozie_share_lib(r, nn_instance.hostname)
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
nn_instance.hostname)
LOG.info('Cluster %s has been started successfully' % cluster.name)
self._set_cluster_info(cluster)
