def _configure_swift(client, cluster): swift_enable = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT) if swift_enable: swift_configs = swift.get_swift_configs() for conf in swift_configs: client.params.hadoop.add(conf['name'], conf['value'])
def _setup_oozie(cluster): with (u.get_oozie(cluster)).remote() as r: LOG.info("Oozie: add hadoop libraries to java.library.path") r.execute_command( "sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libhadoop.so" " /usr/lib64/ && " "sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so" " /usr/lib64/") ext22 = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.OOZIE_EXT22_URL) if ext22: LOG.info("Oozie: downloading and installing ext 2.2 from '%s'" % ext22) r.execute_command( "curl -L -o ext-2.2.zip %s && " "sudo unzip ext-2.2.zip -d /var/lib/oozie && " "rm ext-2.2.zip" % ext22) LOG.info("Oozie: installing oozie share lib") r.execute_command( "mkdir /tmp/oozielib && " "tar xzf /usr/lib/oozie/oozie-sharelib.tar.gz -C /tmp/oozielib && " "rm /tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar &&" "cp /usr/lib/pig/pig-0.11.1-Intel.jar " "/tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar && " "sudo su - -c '" "hadoop fs -put /tmp/oozielib/share /user/oozie/share' hadoop && " "rm -rf /tmp/oozielib")
def _setup_oozie(cluster): with (u.get_oozie(cluster)).remote() as r: LOG.info("Oozie: add hadoop libraries to java.library.path") r.execute_command( "sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libhadoop.so" " /usr/lib64/ && " "sudo ln -s /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so" " /usr/lib64/") ext22 = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.OOZIE_EXT22_URL) if ext22: LOG.info("Oozie: downloading and installing ext 2.2 from '%s'" % ext22) r.execute_command("curl -L -o ext-2.2.zip %s && " "sudo unzip ext-2.2.zip -d /var/lib/oozie && " "rm ext-2.2.zip" % ext22) LOG.info("Oozie: installing oozie share lib") r.execute_command( "mkdir /tmp/oozielib && " "tar xzf /usr/lib/oozie/oozie-sharelib.tar.gz -C /tmp/oozielib && " "rm /tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar &&" "cp /usr/lib/pig/pig-0.11.1-Intel.jar " "/tmp/oozielib/share/lib/pig/pig-0.11.1-Intel.jar && " "sudo su - -c '" "hadoop fs -put /tmp/oozielib/share /user/oozie/share' hadoop && " "rm -rf /tmp/oozielib")
def configure_os_from_instances(cluster, instances): for instance in instances: with instance.remote() as remote: LOG.debug("Configuring OS settings on %s : ", instance.hostname()) # configure hostname, RedHat/Centos specific remote.replace_remote_string('/etc/sysconfig/network', 'HOSTNAME=.*', 'HOSTNAME=%s' % instance.fqdn()) # disable selinux and iptables, because Intel distribution requires # this to be off remote.execute_command('sudo /usr/sbin/setenforce 0') remote.replace_remote_string('/etc/selinux/config', 'SELINUX=.*', 'SELINUX=disabled') # disable iptables remote.execute_command('sudo /sbin/service iptables stop') remote.execute_command('sudo /sbin/chkconfig iptables off') # create 'hadoop' user remote.write_files_to({ 'id_rsa': cluster.extra.get('hadoop_private_ssh_key'), 'authorized_keys': cluster.extra.get('hadoop_public_ssh_key') }) remote.execute_command( 'sudo useradd hadoop && ' 'sudo sh -c \'echo "hadoop ALL=(ALL) NOPASSWD:ALL" ' '>> /etc/sudoers\' && ' 'sudo mkdir -p /home/hadoop/.ssh/ && ' 'sudo mv id_rsa authorized_keys /home/hadoop/.ssh && ' 'sudo chown -R hadoop:hadoop /home/hadoop/.ssh && ' 'sudo chmod 600 /home/hadoop/.ssh/{id_rsa,authorized_keys}') swift_enable = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT) if swift_enable: hadoop_swiftfs_jar_url = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.HADOOP_SWIFTFS_JAR_URL) swift_lib_dir = '/usr/lib/hadoop/lib' swift_lib_path = swift_lib_dir + '/hadoop-swift-latest.jar' cmd = ('sudo mkdir -p %s && sudo curl \'%s\' -o %s' % (swift_lib_dir, hadoop_swiftfs_jar_url, swift_lib_path)) remote.execute_command(cmd)
def install_manager(cluster): LOG.info("Starting Install Manager Process") mng_instance = u.get_instance(cluster, 'manager') idh_tarball_path = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.IDH_TARBALL_URL) idh_tarball_filename = idh_tarball_path.rsplit('/', 1)[-1] idh_dir = idh_tarball_filename[:idh_tarball_filename.find('.tar.gz')] LOG.info("IDH tgz will be retrieved from: \'%s\'", idh_tarball_path) idh_repo = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.IDH_REPO_URL) os_repo = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.OS_REPO_URL) idh_install_cmd = 'sudo ./%s/install.sh --mode=silent 2>&1' % idh_dir with mng_instance.remote() as r: LOG.info("Download IDH manager ") try: r.execute_command('curl -O %s 2>&1' % idh_tarball_path) except Exception as e: raise RuntimeError("Unable to download IDH manager from %s" % idh_tarball_path, e) # unpack archive LOG.info("Unpack manager %s ", idh_tarball_filename) try: r.execute_command('tar xzf %s 2>&1' % idh_tarball_filename) except Exception as e: raise RuntimeError("Unable to unpack tgz %s", idh_tarball_filename, e) # install idh LOG.debug("Install manager with %s : ", idh_install_cmd) inst_conf = _INST_CONF_TEMPLATE % (os_repo, idh_repo) r.write_file_to('%s/ui-installer/conf' % idh_dir, inst_conf) #TODO(alazarev) make timeout configurable (bug #1262897) r.execute_command(idh_install_cmd, timeout=3600) # fix nginx persimmions bug r.execute_command('sudo chmod o+x /var/lib/nginx/ /var/lib/nginx/tmp ' '/var/lib/nginx/tmp/client_body') # waiting start idh manager #TODO(alazarev) make timeout configurable (bug #1262897) timeout = 600 LOG.debug("Waiting %s seconds for Manager to start : ", timeout) while timeout: try: telnetlib.Telnet(mng_instance.management_ip, 9443) break except IOError: timeout -= 2 context.sleep(2) else: message = ("IDH Manager failed to start in %s minutes on node '%s' " "of cluster '%s'" % (timeout / 60, mng_instance.management_ip, cluster.name)) LOG.error(message) raise iex.IntelPluginException(message)
def install_manager(cluster): LOG.info("Starting Install Manager Process") mng_instance = u.get_instance(cluster, 'manager') idh_tarball_path = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.IDH_TARBALL_URL) idh_tarball_filename = idh_tarball_path.rsplit('/', 1)[-1] idh_dir = idh_tarball_filename[:idh_tarball_filename.find('.tar.gz')] LOG.info("IDH tgz will be retrieved from: \'%s\'", idh_tarball_path) idh_repo = c_helper.get_config_value( cluster.cluster_configs.get('general'), c_helper.IDH_REPO_URL) os_repo = c_helper.get_config_value(cluster.cluster_configs.get('general'), c_helper.OS_REPO_URL) idh_install_cmd = 'sudo ./%s/install.sh --mode=silent 2>&1' % idh_dir with mng_instance.remote() as r: LOG.info("Download IDH manager ") try: r.execute_command('curl -O %s 2>&1' % idh_tarball_path) except Exception as e: raise RuntimeError( "Unable to download IDH manager from %s" % idh_tarball_path, e) # unpack archive LOG.info("Unpack manager %s ", idh_tarball_filename) try: r.execute_command('tar xzf %s 2>&1' % idh_tarball_filename) except Exception as e: raise RuntimeError("Unable to unpack tgz %s", idh_tarball_filename, e) # install idh LOG.debug("Install manager with %s : ", idh_install_cmd) inst_conf = _INST_CONF_TEMPLATE % (os_repo, idh_repo) r.write_file_to('%s/ui-installer/conf' % idh_dir, inst_conf) #TODO(alazarev) make timeout configurable (bug #1262897) r.execute_command(idh_install_cmd, timeout=3600) # fix nginx persimmions bug r.execute_command('sudo chmod o+x /var/lib/nginx/ /var/lib/nginx/tmp ' '/var/lib/nginx/tmp/client_body') # waiting start idh manager #TODO(alazarev) make timeout configurable (bug #1262897) timeout = 600 LOG.debug("Waiting %s seconds for Manager to start : ", timeout) while timeout: try: telnetlib.Telnet(mng_instance.management_ip, 9443) break except IOError: timeout -= 2 context.sleep(2) else: message = ("IDH Manager failed to start in %s minutes on node '%s' " "of cluster '%s'" % (timeout / 60, mng_instance.management_ip, cluster.name)) LOG.error(message) raise iex.IntelPluginException(message)