def _configure_swift(client, cluster):
swift_enable = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT)
if swift_enable:
swift_configs = swift.get_swift_configs()
for conf in swift_configs:
client.params.hadoop.add(conf['name'], conf['value'])
def _setup_oozie(cluster):
with (u.get_oozie(cluster)).remote() as r:
LOG.info("Oozie: add hadoop libraries to java.library.path")
r.execute_command(
"sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libhadoop.so"
" /usr/lib64/ && "
"sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so"
" /usr/lib64/")
ext22 = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.OOZIE_EXT22_URL)
if ext22:
LOG.info("Oozie: downloading and installing ext 2.2 from '%s'"
% ext22)
r.execute_command(
"curl -L -o ext-2.2.zip %s && "
"sudo unzip ext-2.2.zip -d /var/lib/oozie && "
"rm ext-2.2.zip" % ext22)
LOG.info("Oozie: installing oozie share lib")
r.execute_command(
"mkdir /tmp/oozielib && "
"tar xzf /usr/lib/oozie/oozie-sharelib.tar.gz -C /tmp/oozielib && "
"rm /tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar &&"
"cp /usr/lib/pig/pig-0.11.1-Intel.jar "
"/tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar && "
"sudo su - -c '"
"hadoop fs -put /tmp/oozielib/share /user/oozie/share' hadoop && "
"rm -rf /tmp/oozielib")
def configure_os_from_instances(cluster, instances):
for instance in instances:
with instance.remote() as remote:
LOG.debug("Configuring OS settings on %s : ", instance.hostname())
# configure hostname, RedHat/Centos specific
remote.replace_remote_string('/etc/sysconfig/network',
'HOSTNAME=.*',
'HOSTNAME=%s' % instance.fqdn())
# disable selinux and iptables, because Intel distribution requires
# this to be off
remote.execute_command('sudo /usr/sbin/setenforce 0')
remote.replace_remote_string('/etc/selinux/config',
'SELINUX=.*', 'SELINUX=disabled')
# disable iptables
remote.execute_command('sudo /sbin/service iptables stop')
remote.execute_command('sudo /sbin/chkconfig iptables off')
# create 'hadoop' user
remote.write_files_to({
'id_rsa': cluster.extra.get('hadoop_private_ssh_key'),
'authorized_keys': cluster.extra.get('hadoop_public_ssh_key')
})
remote.execute_command(
'sudo useradd hadoop && '
'sudo sh -c \'echo "hadoop ALL=(ALL) NOPASSWD:ALL" '
'>> /etc/sudoers\' && '
'sudo mkdir -p /home/hadoop/.ssh/ && '
'sudo mv id_rsa authorized_keys /home/hadoop/.ssh && '
'sudo chown -R hadoop:hadoop /home/hadoop/.ssh && '
'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}')
swift_enable = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT)
if swift_enable:
hadoop_swiftfs_jar_url = c_helper.get_config_value(
cluster.cluster_configs.get('general'),
c_helper.HADOOP_SWIFTFS_JAR_URL)
swift_lib_path = '/usr/lib/hadoop/lib/hadoop-swift-latest.jar'
cmd = ('sudo curl \'%s\' -o %s --create-dirs'
% (hadoop_swiftfs_jar_url, swift_lib_path))
remote.execute_command(cmd)
def install_manager(cluster):
LOG.info("Starting Install Manager Process")
mng_instance = u.get_instance(cluster, 'manager')
idh_tarball_path = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.IDH_TARBALL_URL)
idh_tarball_filename = idh_tarball_path.rsplit('/', 1)[-1]
idh_dir = idh_tarball_filename[:idh_tarball_filename.find('.tar.gz')]
LOG.info("IDH tgz will be retrieved from: \'%s\'", idh_tarball_path)
idh_repo = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.IDH_REPO_URL)
os_repo = c_helper.get_config_value(
cluster.cluster_configs.get('general'), c_helper.OS_REPO_URL)
idh_install_cmd = 'sudo ./%s/install.sh --mode=silent 2>&1' % idh_dir
with mng_instance.remote() as r:
LOG.info("Download IDH manager ")
try:
r.execute_command('curl -O %s 2>&1' % idh_tarball_path)
except Exception as e:
raise RuntimeError("Unable to download IDH manager from %s",
idh_tarball_path, e)
# unpack archive
LOG.info("Unpack manager %s ", idh_tarball_filename)
try:
r.execute_command('tar xzf %s 2>&1' % idh_tarball_filename)
except Exception as e:
raise RuntimeError("Unable to unpack tgz %s",
idh_tarball_filename, e)
# install idh
LOG.debug("Install manager with %s : ", idh_install_cmd)
inst_conf = _INST_CONF_TEMPLATE % (os_repo, idh_repo)
r.write_file_to('%s/ui-installer/conf' % idh_dir, inst_conf)
#TODO(alazarev) make timeout configurable (bug #1262897)
r.execute_command(idh_install_cmd, timeout=3600)
# fix nginx persimmions bug
r.execute_command('sudo chmod o+x /var/lib/nginx/ /var/lib/nginx/tmp '
'/var/lib/nginx/tmp/client_body')
# waiting start idh manager
#TODO(alazarev) make timeout configurable (bug #1262897)
timeout = 600
LOG.debug("Waiting %s seconds for Manager to start : ", timeout)
while timeout:
try:
telnetlib.Telnet(mng_instance.management_ip, 9443)
break
except IOError:
timeout -= 2
context.sleep(2)
else:
message = ("IDH Manager failed to start in %s minutes on node '%s' "
"of cluster '%s'"
% (timeout / 60, mng_instance.management_ip, cluster.name))
LOG.error(message)
raise iex.IntelPluginException(message)
