def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(
config['hostLevelParams']
['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def postgresql_server_install(isSystemd):
Package("postgresql96-server")
Package("postgresql96-contrib")
Execute("postgresql-setup initdb")
import data_analytics_studio
data_analytics_studio.setup_data_analytics_studio_postgresql_server()
# Initialize the DBs for DAS.
postgresql_server_action("start", isSystemd)
data_analytics_studio_initdb()
postgresql_server_action("stop", isSystemd)
def remove_hdp_21(self, env):
"""
During Express Upgrade from HDP 2.1 to any higher version (HDP 2.2 or 2.3), the HDP 2.1 bits must be uninstalled.
This is because /usr/bin/hadoop used to be a shell script in HDP 2.1, but in HDP 2.3 it is
a symlink to <stack-root>/current/hadoop-client/bin/hadoop
so both versions cannot coexist.
"""
Logger.info("Attempting to remove bits for HDP 2.1")
config = Script.get_config()
packages_to_remove = [
"zookeeper", "hadoop", "hadoop-lzo", "hadoop-hdfs",
"hadoop-libhdfs", "hadoop-yarn", "hadoop-client",
"hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc",
"hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig",
"oozie", "oozie-client", "pig", "sqoop", "tez"
"falcon", "storm", "flume", "hbase", "phoenix"
]
packages_to_remove.reverse()
Logger.info("Packages to remove: {0}".format(
" ".join(packages_to_remove)))
for name in packages_to_remove:
Logger.info("Attempting to remove {0}".format(name))
Package(name, action="remove")
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Package(['confluent-kafka-mqtt'])
systemd_content = '''
[Unit]
Description=A MQTT proxy for Apache Kafka
Documentation=http://docs.confluent.io/
After=network.target confluent-kafka.target
[Service]
Type=simple
User=cp-kafka-rest
Group=confluent
Environment="LOG_DIR=/var/log/confluent/kafka-mqtt"
ExecStart=/usr/bin/kafka-mqtt-start /etc/confluent-kafka-mqtt/kafka-mqtt-production.properties
TimeoutStopSec=180
Restart=no
[Install]
WantedBy=multi-user.target
'''
File('/usr/lib/systemd/system/confluent-kafka-mqtt.service',
owner='cp-kafka',
group='confluent',
mode=0644,
content=InlineTemplate(systemd_content))
Execute('systemctl enable confluent-kafka-mqtt')
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
if OSCheck.is_windows_family():
if name[-4:] == ".msi":
#TODO all msis must be located in resource folder of server, change it to repo later
Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location']))
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"),
config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
# RepoInstaller.remove_repos(config)
pass
def setup_atlas_storm():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
PropertiesFile(format('{conf_dir}/{atlas_conf_file}'),
properties=params.atlas_props,
owner=params.storm_user,
group=params.user_group,
mode=0644)
atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"storm")
if os.path.exists(atlas_storm_hook_dir):
storm_extlib_dir = os.path.join(params.storm_component_home_dir,
"extlib")
if os.path.exists(storm_extlib_dir):
src_files = os.listdir(atlas_storm_hook_dir)
for file_name in src_files:
atlas_storm_hook_file_name = os.path.join(
atlas_storm_hook_dir, file_name)
storm_lib_file_name = os.path.join(storm_extlib_dir,
file_name)
if (os.path.isfile(atlas_storm_hook_file_name)):
Link(storm_lib_file_name,
to=atlas_storm_hook_file_name)
def setup_atlas_falcon():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"falcon")
if os.path.exists(atlas_falcon_hook_dir):
Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file),
to=os.path.join(params.atlas_conf_dir,
params.atlas_conf_file))
Directory(params.falcon_webinf_lib,
owner=params.falcon_user,
create_parents=True)
src_files = os.listdir(atlas_falcon_hook_dir)
for file_name in src_files:
atlas_falcon_hook_file_name = os.path.join(
atlas_falcon_hook_dir, file_name)
falcon_lib_file_name = os.path.join(params.falcon_webinf_lib,
file_name)
if (os.path.isfile(atlas_falcon_hook_file_name)):
Link(falcon_lib_file_name, to=atlas_falcon_hook_file_name)
def install_packages(self, env):
"""
List of packages that are required by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
repo_installed = False
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
name = package['name']
type = package['type']
if type.lower() == "tarball":
if name.startswith(os.path.sep):
tarball = name
else:
basedir = env.config.basedir
tarball = os.path.join(basedir, name)
install_location = config['configurations']['global'][
'app_install_dir']
Directory(install_location, action="delete")
Directory(install_location)
Tarball(tarball, location=install_location)
else:
if not repo_installed:
RepoInstaller.install_repos(config)
repo_installed = True
Package(name)
except KeyError:
pass # No reason to worry
def install_packages():
import params
if params.host_sys_prepped:
return
packages = [
'unzip', 'curl', 'jdk', 'openssl-devel', 'openldap-clients',
'nss-pam-ldapd', 'pam_ldap', 'pam_krb5', 'authconfig', 'lzo',
'krb5-workstation', 'krb5-libs', 'libcgroup', 'lz4', 'lz4-devel',
'zstd-devel', 'libisal'
] # , 'java-1.8.0-openjdk-devel']
Package(packages,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(" chkconfig nslcd on")
Execute(" chkconfig nscd on")
if not os.path.exists('/cgroups'):
try:
Execute('umount /sys/fs/cgroup/cpu,cpuacct')
Execute('mkdir -p /cgroups/cpu /cgroups/cpuacct')
Execute(
'mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpu cgroup /cgroups/cpu'
)
Execute(
'mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpuacct cgroup /cgroups/cpuacct'
)
except Exception as e:
print "mount cgroups error"
if not os.path.exists('/cgroups/cpu'):
Execute('mkdir -p /cgroups/cpu')
def install_dl_soft():
lock_file = base_lock_dir + '/install_dl'
if not os.path.exists(lock_file):
packages = ['LightGBM', 'libsvm', 'xgboost']
Package(packages)
Execute(" echo 1 > " + lock_file)
def setup_atlas_sqoop():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"sqoop")
if os.path.exists(atlas_sqoop_hook_dir):
Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
to=os.path.join(params.atlas_conf_dir,
params.atlas_conf_file))
src_files = os.listdir(atlas_sqoop_hook_dir)
for file_name in src_files:
atlas_sqoop_hook_file_name = os.path.join(
atlas_sqoop_hook_dir, file_name)
sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
if (os.path.isfile(atlas_sqoop_hook_file_name)):
Link(sqoop_lib_file_name, to=atlas_sqoop_hook_file_name)
def install_superset():
import params
Directory([params.conf_dir, '/var/run/superset'],
owner=params.superset_user,
group=params.user_group,
mode=0775,
create_parents=True)
Package(['superset'])
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
self.set_version()
def install_atlas_hook_packages(atlas_plugin_package,
atlas_ubuntu_plugin_package, host_sys_prepped,
agent_stack_retry_on_unavailability,
agent_stack_retry_count):
if not host_sys_prepped:
Package(
atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else atlas_plugin_package,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
def install_atlas_hook_packages(atlas_plugin_package,
atlas_ubuntu_plugin_package, host_sys_prepped,
agent_stack_retry_on_unavailability,
agent_stack_retry_count):
if not host_sys_prepped:
# This will install packages like atlas-metadata-${service}-plugin needed for Falcon and Hive.
Package(
atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else atlas_plugin_package,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
def install(self, env):
config = self.get_config()
agent_stack_retry_on_unavailability = bool(
config['ambariLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['ambariLevelParams']['agent_stack_retry_count'])
Package(
"fastdfs.x86_64",
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
self.configure(env)
def setup_atlas_falcon():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
def install_other_packages():
import params
if params.host_sys_prepped:
return
packages = [
'unzip', 'curl', 'jdk', 'java-1.8.0-openjdk-devel', 'openldap-clients',
'nss-pam-ldapd', 'pam_ldap', 'pam_krb5', 'authconfig', 'krb5-libs'
]
Package(packages,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
def setup_atlas_hive():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered.
)
PropertiesFile(format('{hive_config_dir}/client.properties'),
properties = params.atlas_client_props,
owner = params.hive_user,
group = params.user_group,
mode = 0644)
def install_gpu_cuda():
import params
lock_file = base_lock_dir + '/install_cuda'
if not os.path.exists(lock_file):
packages = ['cuda', 'libnccl']
Package(packages)
# cudnn
if not os.path.exists('/usr/local/cuda/include/cudnn.h'):
cudnn_file = 'cudnn.tar.bz2'
Execute('wget ' + params.download_url_base + '/' + cudnn_file + ' -O /tmp/' + cudnn_file)
Execute('cd /tmp && tar -jxvf ' + cudnn_file + ' -C /usr/local/cuda/')
Execute(" echo 1 > " + lock_file)
def hook(self, env):
import params
env.set_params(params)
Package(['wget'])
install_repo()
install_packages()
ln_jvmso()
ln_crypto()
ldap_client_conf()
setup_users()
if params.has_namenode:
setup_hadoop_env()
setup_java()
def setup_atlas_falcon():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
PropertiesFile(format('{falcon_conf_dir}/{atlas_conf_file}'),
properties = params.atlas_props,
owner = params.falcon_user,
group = params.user_group,
mode = 0644)
def install_packages():
lock_file = base_lock_dir + '/install_packages'
if not os.path.exists(lock_file):
packages = [
'tar', 'curl', 'wget', 'java-1.8.0-openjdk-devel',
'java-11-openjdk-devel', 'bzip2-libs', 'snappy', 'lzo', 'zlib',
'lz4', 'libzstd', 'libisal', 'openssl-libs', 'openssh-ldap',
'authconfig', 'openldap-clients', 'nss-pam-ldapd', 'pam_krb5',
'sssd-ldap', 'nscd', 'numactl'
]
Package(packages)
Execute(" chkconfig nslcd on")
Execute(" chkconfig nscd on")
Execute('yum remove -y postfix mariadb-libs mariadb')
Execute(" echo 1 > " + lock_file)
def install_packages():
import params
if params.host_sys_prepped:
return
packages = ['unzip', 'curl']
if params.stack_version_formatted != "" and compare_versions(
params.stack_version_formatted, '2.2') >= 0:
stack_selector_package = stack_tools.get_stack_tool_package(
stack_tools.STACK_SELECTOR_NAME)
packages.append(stack_selector_package)
Package(packages,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
def actionexecute(self, env):
config = Script.get_config()
packages_to_remove = config['roleParams']['package_list'].split(',')
structured_output = {'success': [], 'failure': []}
for package_name in packages_to_remove:
try:
Package(package_name,
action='remove',
ignore_dependencies=True)
Logger.info('Removed {0}'.format(package_name))
structured_output['success'].append(package_name)
except Exception, e:
Logger.exception('Failed to remove {0}: {1}'.format(
package_name, str(e)))
structured_output['failure'].append(package_name)
def setup_atlas_hive(configuration_directory=None):
import params
if params.has_atlas:
if configuration_directory is None:
configuration_directory = format("{hive_config_dir}")
if not params.host_sys_prepped:
Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered.
)
PropertiesFile(format('{configuration_directory}/client.properties'),
properties = params.atlas_client_props,
owner = params.hive_user,
group = params.user_group,
mode = 0644)
def install_packages(self, env):
"""
List of packages that are required by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
name = package['name']
Package(name)
except KeyError:
pass # No reason to worry
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
if OSCheck.is_windows_family():
if name[-4:] == ".msi":
#TODO all msis must be located in resource folder of server, change it to repo later
Msi(name,
http_source=os.path.join(
config['hostLevelParams']
['jdk_location']))
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
os.path.join(config['hostLevelParams']['jdk_location'],
"hdp.msi"),
config["hostLevelParams"]["agentCacheDir"], "hdp.msi",
hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
pass
def install_lzo_if_needed():
"""
Install lzo package if {#should_install_lzo} is true
"""
if not should_install_lzo():
return
lzo_packages = get_lzo_packages()
config = Script.get_config()
agent_stack_retry_on_unavailability = config['hostLevelParams'][
'agent_stack_retry_on_unavailability']
agent_stack_retry_count = expect(
"/hostLevelParams/agent_stack_retry_count", int)
Package(lzo_packages,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
def install_lzo_if_needed():
"""
Install lzo package if {#should_install_lzo} is true
"""
if not should_install_lzo():
return
# If user has just accepted GPL license. GPL repository can not yet be present.
Script.repository_util.create_repo_files()
lzo_packages = get_lzo_packages()
config = Script.get_config()
agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
Package(lzo_packages,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
)
