def install_apollo(): import params Directory( [params.pid_dir, params.log_dir, params.conf_dir + '/adminservice'], owner=params.apollo_user, group=params.user_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params. version_dir_adminservice) or not os.path.exists( params.install_dir_adminservice): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir_adminservice) Execute('rm -rf %s' % params.install_dir_adminservice) Execute('/bin/rm -f /tmp/' + params.filename_adminservice) Execute('wget ' + params.download_url_adminservice + ' -O /tmp/' + params.filename_adminservice, user=params.apollo_user) Execute('unzip /tmp/' + params.filename_adminservice + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir_adminservice + ' ' + params.install_dir_adminservice) Execute('cp -r ' + params.install_dir_adminservice + '/config/* ' + params.conf_dir + '/adminservice') Execute('rm -rf ' + params.install_dir_adminservice + '/config') Execute('ln -s ' + params.conf_dir + '/adminservice' + ' ' + params.install_dir_adminservice + '/config') Execute('chown -R %s:%s %s/%s' % (params.apollo_user, params.user_group, Script.get_stack_root(), params.version_dir_adminservice)) Execute('chown -R %s:%s %s' % (params.apollo_user, params.user_group, params.install_dir_adminservice))
def install_ranger(self): import os, params Directory([params.ranger_conf, params.admin_log_dir], owner=params.unix_user, group=params.user_group, mode=0775, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir_admin) or not os.path.exists( params.install_dir_admin): Execute('rm -rf %s' % params.install_dir_admin) Execute('wget ' + params.download_url_admin + ' -O /tmp/' + params.filename_admin, user=params.unix_user) Execute('tar -zxf /tmp/' + params.filename_admin + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir_admin + ' ' + params.install_dir_admin) Execute(' rm -rf ' + params.install_dir_admin + '/conf') Execute(' rm -rf ' + params.ranger_conf + '/conf/*') Execute(' rm -rf ' + params.install_dir_admin + '/ews/webapp/WEB-INF/classes/conf') Execute('ln -s ' + params.ranger_conf + ' ' + params.install_dir_admin + '/conf') Execute('ln -s ' + params.ranger_conf + ' ' + params.install_dir_admin + '/ews/webapp/WEB-INF/classes/conf') Execute('ln -s ' + params.admin_log_dir + ' ' + params.install_dir_admin + '/ews/logs') Execute('chown -R %s:%s %s/%s' % (params.unix_user, params.user_group, Script.get_stack_root(), params.version_dir_admin)) Execute('chown -R %s:%s %s' % (params.unix_user, params.user_group, params.install_dir_admin)) Execute('/bin/rm -f /tmp/' + params.filename_admin)
def install_pika(): import params Directory([ params.pid_dir, params.log_dir, params.conf_dir, params.db_dir, params.dump_dir ], owner=params.pika_user, group=params.pika_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.pika_user) Execute('tar -jxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s %s/output %s ' % (params.stack_root, params.version_dir)) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute('mkdir ' + params.install_dir + '/log && chmod 777 ' + params.install_dir + '/log') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/pika.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.pika_user, params.pika_group, Script.get_stack_root(), params.version_dir)) Execute('chown -R %s:%s %s' % (params.pika_user, params.pika_group, params.install_dir))
def install_rocketmq(): import params Directory([ params.pid_dir, params.log_dir, params.conf_dir, params.store_commitlog, params.store_queue ], owner=params.rocketmq_user, group=params.user_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.rocketmq_user) Execute('tar -zxvf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute('rm -rf ' + params.install_dir + '/conf ') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir + '/conf ') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/rocketmq.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.rocketmq_user, params.user_group, Script.get_stack_root(), params.version_dir)) Execute('chown -R %s:%s %s' % (params.rocketmq_user, params.user_group, params.install_dir))
def install_nifi_toolkit(): import params Directory([params.toolkit_tmp_dir, params.nifi_toolkit_conf_dir], owner=params.nifi_user, group=params.nifi_group, mode=0775, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.toolkit_version_dir) or not os.path.exists( params.toolkit_install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.toolkit_version_dir) Execute('rm -rf %s' % params.toolkit_install_dir) Execute('wget ' + params.toolkit_download_url + ' -O /tmp/' + params.toolkit_filename, user=params.nifi_user) Execute('tar -zxf /tmp/' + params.toolkit_filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.toolkit_version_dir + ' ' + params.toolkit_install_dir) Execute(' cp -rf ' + params.toolkit_install_dir + '/conf/* ' + params.nifi_toolkit_conf_dir) Execute(' rm -rf ' + params.toolkit_install_dir + '/conf') Execute('ln -s ' + params.nifi_toolkit_conf_dir + ' ' + params.toolkit_install_dir + '/conf') Execute('chown -R %s:%s %s/%s' % (params.nifi_user, params.nifi_group, Script.get_stack_root(), params.toolkit_version_dir)) Execute( 'chown -R %s:%s %s' % (params.nifi_user, params.nifi_group, params.toolkit_install_dir)) Execute('/bin/rm -f /tmp/' + params.toolkit_filename)
def install_ranger(self): import os, params Directory([params.ranger_tagsync_conf], owner=params.unix_user, group=params.user_group, mode=0775, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params. version_dir_tagsync) or not os.path.exists( params.install_dir_tagsync): Execute('rm -rf %s' % params.install_dir_tagsync) Execute('wget ' + params.download_url_tagsync + ' -O /tmp/' + params.filename_tagsync, user=params.unix_user) Execute('tar -zxf /tmp/' + params.filename_tagsync + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir_tagsync + ' ' + params.install_dir_tagsync) Execute(' rm -rf ' + params.install_dir_tagsync + '/conf') Execute('ln -s ' + params.ranger_tagsync_conf + ' ' + params.install_dir_tagsync + '/conf') Execute('ln -s ' + params.tagsync_log_dir + ' ' + params.install_dir_tagsync + '/logs') Execute('chown -R %s:%s %s/%s' % (params.unix_user, params.user_group, Script.get_stack_root(), params.version_dir_tagsync)) Execute('chown -R %s:%s %s' % (params.unix_user, params.user_group, params.install_dir_tagsync)) Execute('/bin/rm -f /tmp/' + params.filename_tagsync)
def install_wherehows(): import params Directory([ params.log_dir, params.conf_dir + '/backend', params.conf_dir + '/jobs', '/var/run/wherehows' ], owner=params.wherehows_user, group=params.user_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir_backend ) or not os.path.exists(params.install_dir_backend): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir_backend) Execute('rm -rf %s' % params.install_dir_backend) Execute('/bin/rm -f /tmp/' + params.filename_backend) Execute( 'wget ' + params.download_url_backend + ' -O /tmp/' + params.filename_backend, user=params.wherehows_user) Execute('tar -zxf /tmp/' + params.filename_backend + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir_backend + ' ' + params.install_dir_backend) Execute('rm -rf %s/logs && ln -s %s %s/logs ' % (params.install_dir_backend, params.log_dir, params.install_dir_backend)) Execute('rm -rf %s/conf && ln -s %s %s/conf ' % (params.install_dir_backend, params.conf_dir, params.install_dir_backend)) Execute('chown -R %s:%s %s/%s' % (params.wherehows_user, params.user_group, params.stack_root,params.version_dir_backend)) Execute( 'chown -R %s:%s %s' % (params.wherehows_user, params.user_group, params.install_dir_backend))
def install_beacon(): import params Directory([params.etc_prefix_dir], owner=params.beacon_user, group=params.user_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute( 'wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.beacon_user) Execute('tar -zxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute(' cp -r ' + params.install_dir + '/conf/* ' + params.etc_prefix_dir) Execute(' rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.etc_prefix_dir + ' ' + params.install_dir + '/conf') Execute('chown -R %s:%s %s/%s' % (params.beacon_user, params.user_group, params.stack_root, params.version_dir)) Execute('chown -R %s:%s %s' % (params.beacon_user, params.user_group, params.install_dir)) Execute('/bin/rm -f /tmp/' + params.filename)
def install_dbus(): import params Directory([params.pid_dir, params.log_dir, params.conf_dir], owner=params.dbus_user, group=params.dbus_group, mode=0755, create_parents=True) params.HdfsResource('/user/dbus', type="directory", action="create_on_execute", owner=params.dbus_user, mode=0755) params.HdfsResource('/dbus', type="directory", action="create_on_execute", owner=params.dbus_user, mode=0755) params.HdfsResource(None, action="execute") # copy_to_hdfs( # "udf", # params.user_group, # params.hdfs_user, # custom_src_file = '', # custom_dest_file='/dbus/udf/abc.jar', # replace_existing_files=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.dbus_user) Execute('tar -jxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute('cp -r ' + params.install_dir + '/conf/* ' + params.conf_dir) Execute('rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir + '/conf') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/dbus.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.dbus_user, params.dbus_group, params.stack_root, params.version_dir)) Execute('chown -R %s:%s %s' % (params.dbus_user, params.dbus_group, params.install_dir))
def install_xlearning(): import params Directory([params.conf_dir, params.log_dir], owner=params.xlearning_user, group=params.xlearning_group, mode=0755, create_parents=True) params.HdfsResource('/xlearning/staging', type="directory", action="create_on_execute", owner=params.xlearning_user, mode=0755) params.HdfsResource('/xlearning/eventlog', type="directory", action="create_on_execute", owner=params.xlearning_user, mode=0755) params.HdfsResource('/xlearning/history', type="directory", action="create_on_execute", owner=params.xlearning_user, mode=0755) params.HdfsResource(None, action="execute") if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.xlearning_user) Execute('tar -zxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute(' cp -r ' + params.install_dir + '/conf/* ' + params.conf_dir) Execute(' rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir + '/conf') Execute('ln -s ' + params.log_dir + ' ' + params.install_dir + '/logs/xlearning') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/xlearning.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.xlearning_user, params.xlearning_group, params.stack_root, params.version_dir)) Execute('chown -R %s:%s %s' % (params.xlearning_user, params.xlearning_group, params.install_dir)) Execute('/bin/rm -f /tmp/' + params.filename)
def install_wormhole(): import params Directory([params.pid_dir, params.log_dir, params.conf_dir], owner=params.wormhole_user, group=params.wormhole_group, mode=0755, create_parents=True) params.HdfsResource('/user/wormhole', type="directory", action="create_on_execute", owner=params.wormhole_user, mode=0755) params.HdfsResource('/wormhole', type="directory", action="create_on_execute", owner=params.wormhole_user, mode=0755) params.HdfsResource('/wormhole/udfjars', type="directory", action="create_on_execute", owner=params.wormhole_user, mode=0755) params.HdfsResource(None, action="execute") if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.wormhole_user) Execute('tar -jxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute('cp -r ' + params.install_dir + '/conf/* ' + params.conf_dir) Execute('rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir + '/conf') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/wormhole.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.wormhole_user, params.wormhole_group, Script.get_stack_root(), params.version_dir)) Execute( 'chown -R %s:%s %s' % (params.wormhole_user, params.wormhole_group, params.install_dir))
def install_graphouse(): import params Directory([params.graphouse_conf_dir, graphouse_log_dir], owner=params.graphite_user, group=params.user_group, mode=0775, cd_access="a", create_parents=True) File('/tmp/init_clickhouse.sql', content=StaticFile("init_clickhouse.sql"), mode=0755) # todo excute clickhouse.sql if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.graphite_user) Execute('tar -xf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute(' rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.graphouse_conf_dir + ' ' + params.install_dir + '/conf') Execute(' rm -rf ' + params.install_dir + '/log') Execute('ln -s ' + params.log_dir + ' ' + params.install_dir + '/log') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/graphouse.sh" % params.install_dir) Execute('chown -R %s:%s %s/%s' % (params.graphite_user, params.user_group, params.stack_root, params.version_dir)) Execute('chown -R %s:%s %s' % (params.graphite_user, params.user_group, params.install_dir)) Execute('/bin/rm -f /tmp/' + params.filename) File('/etc/init.d/graphouse', content=StaticFile("graphouse.init"), mode=0755) Execute('chkconfig graphouse on')
def install_azkaban(): import params Directory([params.conf_dir, params.log_dir], owner=params.azkaban_user, group=params.user_group, mode=0755, create_parents=True) File(params.install_dir_executor + '/lib/azkaban-ldap-usermanager-1.2.1-SNAPSHOT.jar', content=StaticFile("azkaban-ldap-usermanager-1.2.1-SNAPSHOT.jar"), mode=0755) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir_executor) or not os.path.exists( params.install_dir_executor): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir_executor) Execute('rm -rf %s' % params.install_dir_executor) Execute('wget ' + params.download_url_executor + ' -O /tmp/' + params.filename_executor, user=params.azkaban_user) Execute('tar -zxf /tmp/' + params.filename_executor + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir_executor + ' ' + params.install_dir_executor) Execute(' cp -r ' + params.install_dir_executor + '/conf/* ' + params.conf_dir) Execute(' rm -rf ' + params.install_dir_executor + '/conf') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir_executor + '/conf') Execute('ln -s ' + params.log_dir + ' ' + params.install_dir_executor + '/logs/azkaban') Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/azkaban.sh" % params.install_dir_executor) Execute('chown -R %s:%s %s/%s' % (params.azkaban_user, params.user_group, Script.get_stack_root(), params.version_dir_executor)) Execute('chown -R %s:%s %s' % (params.azkaban_user, params.user_group, params.install_dir_executor)) Execute('/bin/rm -f /tmp/' + params.filename_executor)
def install_registry(): import params Directory(['/data1/registry', '/etc/harbor'], mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename) Execute('tar -jxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) File('/etc/harbor/harbor.cfg', content=InlineTemplate(params.harbor_cfg_content), mode=0755) Execute(params.install_dir + '/install.sh')
def install_nifi_registry(): import params Directory([params.nifi_registry_config_dir], owner=params.nifi_user, group=params.nifi_group, mode=0775, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.registry_version_dir) or not os.path.exists( params.registry_install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.registry_version_dir) Execute('rm -rf %s' % params.registry_install_dir) Execute('wget ' + params.registry_download_url + ' -O /tmp/' + params.registry_filename, user=params.nifi_registry_user) Execute('tar -zxf /tmp/' + params.registry_filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.registry_version_dir + ' ' + params.registry_install_dir) Execute(' cp -rf ' + params.registry_install_dir + '/conf/* ' + params.registry_install_dir) Execute(' rm -rf ' + params.registry_install_dir + '/conf') Execute('ln -s ' + params.registry_install_dir + ' ' + params.registry_install_dir + '/conf') Execute('chown -R %s:%s %s/%s' % (params.nifi_registry_user, params.nifi_registry_group, Script.get_stack_root(), params.registry_version_dir)) Execute('chown -R %s:%s %s' % (params.nifi_registry_user, params.nifi_registry_group, params.registry_install_dir)) Execute('/bin/rm -f /tmp/' + params.registry_filename) Execute('export JAVA_HOME=' + params.jdk64_home + ';' + params.nifi_registry_bin_dir + '/nifi-registry.sh install >> ' + params.nifi_registry_log_file, user=params.nifi_registry_user)
def install_sonar(): import params if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('wget ' + params.download_url + ' -O /tmp/' + params.filename, user='******') Execute('cd %s ; unzip /tmp/%s' % (params.stack_root, params.filename)) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute(' mkdir -p ' + params.conf_dir + ' && cp -r ' + params.install_dir + '/conf/* ' + params.conf_dir) Execute(' cp -r ' + params.install_dir + '/conf/* ' + params.conf_dir) Execute(' rm -rf ' + params.install_dir + '/conf') Execute('ln -s ' + params.conf_dir + ' ' + params.install_dir + '/conf') Execute('/bin/rm -f /tmp/' + params.filename)
def install_doris(): import params Directory([params.pid_dir, params.log_dir], owner=params.doris_user, group=params.doris_group, mode=0755, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/' + params.version_dir) or not os.path.exists( params.install_dir): Execute('rm -rf %s' % Script.get_stack_root() + '/' + params.version_dir) Execute('rm -rf %s' % params.install_dir) Execute('/bin/rm -f /tmp/' + params.filename) Execute( 'wget ' + params.download_url + ' -O /tmp/' + params.filename, user=params.doris_user) Execute('tar -zxf /tmp/' + params.filename + ' -C ' + Script.get_stack_root()) Execute('ln -s ' + Script.get_stack_root() + '/' + params.version_dir + ' ' + params.install_dir) Execute('mkdir ' + params.install_dir + '/log && chmod 777 ' + params.install_dir + '/log') Execute('chown -R %s:%s %s/%s' % (params.doris_user, params.doris_group,Script.get_stack_root(), params.version_dir)) Execute('chown -R %s:%s %s' % (params.doris_user, params.doris_group, params.install_dir))
def setup_atlas_jar_symlinks(hook_name, jar_source_dir): """ In HDP 2.3, 2.4, and 2.5.0.0, Sqoop and Storm still relied on the following method to setup Atlas hooks because the RPM for Sqoop and Storm did not bring in any dependencies. /usr/hdp/current/storm-*/libext/ should contain symlinks for every jar in /usr/hdp/current/atlas-server/hooks/storm/somejavafile.jar /usr/hdp/current/sqoop-*/lib/ should contain symlinks for every jar in /usr/hdp/current/atlas-server/hooks/sqoop/somejavafile.jar In HDP 2.5.x.y, we plan to have the Sqoop and Storm rpms have additional dependencies on some sqoop-atlas-hook and storm-atlas-hook rpms, respectively, that will bring in the necessary jars and create the symlinks. If atlas is present on this host, then link the jars from {stack_root}/current/{hook_name}/lib/name_version.jar -> {jar_source_dir}/name_version.jar @param hook_name: one of sqoop, storm @param jar_source_dir: directory of where the symlinks need to be created from. """ import params stack_root = Script.get_stack_root() atlas_home_dir = os.path.join(stack_root, "current", "atlas-server") # if this is an upgrade/downagrade, then we must link in the correct version # which may not be "current", so change the home directory location upgrade_type = Script.get_upgrade_type( default("/commandParams/upgrade_type", "")) if upgrade_type is not None: version_dir_segment = stack_features.get_stack_feature_version( Script.get_config()) atlas_home_dir = os.path.join(stack_root, version_dir_segment, "atlas") # Will only exist if this host contains Atlas Server atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name) if os.path.exists(atlas_hook_dir): Logger.info( "Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done." % (jar_source_dir, atlas_hook_dir)) src_files = os.listdir(atlas_hook_dir) for file_name in src_files: atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name) source_lib_file_name = os.path.join(jar_source_dir, file_name) if os.path.isfile(atlas_hook_file_name): Link(source_lib_file_name, to=atlas_hook_file_name) else: Logger.info("Atlas hook directory path {0} doesn't exist".format( atlas_hook_dir))
def install_graphite_api(): import params Directory([ params.graphite_conf_dir, params.log_dir, params.pid_dir, '/srv/graphite' ], owner=params.graphite_user, group=params.user_group, mode=0775, cd_access="a", create_parents=True) if not os.path.exists(params.install_dir_graphite_api): Execute('wget ' + params.download_url_graphite_api + ' -O /tmp/' + params.filename_graphite_api, user=params.graphite_user) Execute('tar -xf /tmp/' + params.filename_graphite_api + ' -C ' + Script.get_stack_root()) Execute('chown -R %s:%s %s' % (params.graphite_user, params.user_group, params.install_dir_graphite_api)) Execute('/bin/rm -f /tmp/' + params.filename_graphite_api) File( params.install_dir_graphite_api + '/lib/python3.6/site-packages/graphite_api/finders/graphouse_api.py', content=StaticFile("graphouse_api.py"), mode=0755) File('/usr/lib/systemd/system/graphite-api.service', content=params.graphite_api_systemd_content, mode=0755) File('/etc/sysconfig/memcached', content=params.memcached_content, mode=0755) Execute('systemctl daemon-reload') Execute('systemctl enable graphite-api') Execute('systemctl enable memcached') Execute('systemctl start memcached')
def install_graphite_web(): import params Directory([params.graphite_conf_dir, params.log_dir, params.pid_dir], owner=params.graphite_user, group=params.user_group, mode=0775, cd_access="a", create_parents=True) if not os.path.exists(params.install_dir_graphite_web): Execute('wget ' + params.download_url_graphite_web + ' -O /tmp/' + params.filename_graphite_web, user=params.graphite_user) Execute('tar -xf /tmp/' + params.filename_graphite_web + ' -C ' + Script.get_stack_root()) Execute(' rm -rf ' + params.install_dir_graphite_web + '/conf') Execute('ln -s ' + params.graphite_conf_dir + ' ' + params.install_dir_graphite_web + '/conf') Execute('chown -R %s:%s %s' % (params.graphite_user, params.user_group, params.install_dir_graphite_web)) Execute('/bin/rm -f /tmp/' + params.filename_graphite_web) File(params.install_dir_graphite_web + '/webapp/graphite/graphouse.py', content=StaticFile("graphouse.py"), mode=0755) File('/usr/lib/systemd/system/graphite-web.service', content=params.graphite_web_systemd_content, mode=0755) File('/etc/sysconfig/memcached', content=params.memcached_content, mode=0755) Execute('systemctl daemon-reload') Execute('systemctl enable graphite-web') Execute('systemctl enable memcached') Execute('systemctl start memcached')
def status(self, env): import params env.set_params(params) check_process_status(Script.get_stack_root() + '/atlassian/jira/work/jira.pid')
False) # ranger storm plugin end section namenode_hosts = default("/clusterHostInfo/namenode_hosts", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env'][ 'hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env'][ 'hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site'][ 'fs.defaultFS'] if has_namenode else None hadoop_bin_dir = Script.get_stack_root( ) + '/hadoop/bin/' if has_namenode else None hadoop_conf_dir = '/etc/hadoop' if has_namenode else None kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) dfs_type = default("/clusterLevelParams/dfs_type", "") import functools # create partial functions with common arguments for every HdfsResource call # to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file= "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled=security_enabled,
limitations under the License. """ import os from resource_management.libraries.script import Script from resource_management.libraries.functions.format import format from resource_management.libraries.functions.default import default from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.core.exceptions import Fail from resource_management.libraries.functions import get_kinit_path config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() current_host = config['agentLevelParams']['hostname'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) ugsync_keytab = config['configurations']['ranger-ugsync-site'][ 'ranger.usersync.kerberos.keytab'] ugsync_principal = config['configurations']['ranger-ugsync-site'][ 'ranger.usersync.kerberos.principal'].replace('_HOST', current_host.lower()) install_dir_admin = Script.get_stack_root() + "/ranger-admin" download_url_admin = default("/configurations/ranger-env/download_url_admin", "") filename_admin = download_url_admin.split('/')[-1] version_dir_admin = filename_admin.replace('.tar.gz', '').replace('.tgz', '')
jdk_location = config['ambariLevelParams']['jdk_location'] dfs_type = default("/clusterLevelParams/dfs_type", "") jdbc_jar_name = default("/ambariLevelParams/custom_mysql_jdbc_name", None) hostname = config['agentLevelParams']['hostname'] ambari_cluster_name = config['clusterName'] java_version = expect("/hostLevelParams/java_version", int) host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) beacon_hosts = default("/clusterHostInfo/beacon_server_hosts", None) if type(beacon_hosts) is list: beacon_host_name = beacon_hosts[0] else: beacon_host_name = beacon_hosts tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() install_dir = stack_root + '/canal' download_url = config['configurations']['beacon-env']['download_url'] filename = download_url.split('/')[-1] version_dir = filename.replace('.tar.gz', '').replace('.tgz', '') beacon_user = config['configurations']['beacon-env']['canal_user'] beacon_home_dir = stack_root + '/beacon' beacon_root = beacon_home_dir beacon_webapp_dir = beacon_home_dir + '/webapp' beacon_home = beacon_home_dir beacon_cluster_name = format('{ambari_cluster_name}') credential_store_enabled = False if 'credentialStoreEnabled' in config: credential_store_enabled = config['credentialStoreEnabled']
import os from ambari_commons.constants import AMBARI_SUDO_BINARY from ambari_commons.constants import LOGFEEDER_CONF_DIR from resource_management.libraries.script import Script from resource_management.libraries.script.script import get_config_lock_file from resource_management.libraries.functions import default from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_stack_version, get_major_version from string import lower config = Script.get_config() tmp_dir = Script.get_tmp_dir() versioned_stack_root = Script.get_stack_root() dfs_type = default("/clusterLevelParams/dfs_type", "") is_parallel_execution_enabled = int( default("/agentConfigParams/agent/parallel_execution", 0)) == 1 host_sys_prepped = default("/ambariLevelParams/host_sys_prepped", False) sudo = AMBARI_SUDO_BINARY stack_version_unformatted = config['clusterLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) major_stack_version = get_major_version(stack_version_formatted) hadoop_home = versioned_stack_root + '/hadoop' hadoop_libexec_dir = hadoop_home + "/libexec"
]) NON_CLIENT_SERVICES = [ SERVICE.HIVE, SERVICE.STORM, SERVICE.FALCON, SERVICE.HBASE ] from resource_management.core.resources import Execute download_url_base = default("/configurations/cluster-env/download_url_base", 'http://assets.example.com/') CONST_ATLAS_VERSION = default("/configurations/atlas-env/plugin_version", '1.0.0') CONST_DOWNLOAD_URL_BASE = download_url_base + '/atlas/' stack_root = Script.get_stack_root().replace('/usr/hdp', '/opt') def install_atlas_hook(hook_name): if not hook_name: return '' filename = 'atlas-' + CONST_ATLAS_VERSION + '-' + hook_name + '-plugin.tar.gz' download_url = CONST_DOWNLOAD_URL_BASE + filename version_dir = stack_root + '/atlas-' + CONST_ATLAS_VERSION + '-' + hook_name + '-plugin' install_dir = stack_root + '/atlas-' + hook_name + '-plugin' if not os.path.exists(version_dir): Execute('rm -rf %s' % install_dir) Execute('wget ' + download_url + ' -O /tmp/' + filename) Execute('tar -zxf /tmp/' + filename + ' -C ' + stack_root) Execute('ln -s ' + version_dir + ' ' + install_dir)
rest_hbase_configured_flag_file = status_params.rest_hbase_configured_flag_file rest_hbase_acl_configured_flag_file = status_params.rest_hbase_acl_configured_flag_file metron_knox_installed_flag_file = status_params.metron_knox_installed_flag_file global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] # Elasticsearch hosts and port management es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] es_hosts = config['configurations']['metron-env']['es_hosts'] es_host_list = es_hosts.split(",") es_http_port = config['configurations']['metron-env']['es_http_port'] es_url = ",".join([host + ":" + es_http_port for host in es_host_list]) es_http_url = es_host_list[0] + ":" + es_http_port es_date_format = config['configurations']['metron-env']['es_date_format'] # hadoop params stack_root = Script.get_stack_root() # This is the cluster group named 'hadoop'. Its membership is the stack process user ids not individual users. # The config name 'user_group' is out of our control and a bit misleading, so it is renamed to 'hadoop_group'. hadoop_group = config['configurations']['cluster-env']['user_group'] hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() kafka_home = os.path.join(stack_root, "current", "kafka-broker") kafka_bin_dir = os.path.join(kafka_home, "bin") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) has_zk_host = not len(zk_hosts) == 0 zookeeper_quorum = None if has_zk_host: if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
def install_jira(): import params Directory([params.conf_dir], owner='jira', mode=0775, create_parents=True) if not os.path.exists(Script.get_stack_root() + '/atlassian/jira'): Execute('wget ' + params.download_url_jira + ' -O /tmp/jira.bin') Execute('sudo /tmp/jira.bin')
from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions.get_architecture import get_architecture from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.libraries.functions.namenode_ha_utils import get_properties_for_all_nameservices, namenode_federation_enabled config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root = Script.get_stack_root() architecture = get_architecture() dfs_type = default("/clusterLevelParams/dfs_type", "") artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jdk_name = default("/ambariLevelParams/jdk_name", None) java_home = config['ambariLevelParams']['java_home'] java_version = expect("/ambariLevelParams/java_version", int) jdk_location = config['ambariLevelParams']['jdk_location'] hadoop_custom_extensions_enabled = default( "/configurations/core-site/hadoop.custom-extensions.enabled", False) sudo = AMBARI_SUDO_BINARY
See the License for the specific language governing permissions and limitations under the License. """ import os import sys from resource_management.libraries.functions.default import default from resource_management.libraries.functions.get_kinit_path import get_kinit_path from resource_management.libraries.script import Script from resource_management.libraries.functions.format import format from resource_management.libraries.functions.expect import expect from resource_management.core.exceptions import Fail config = Script.get_config() stack_root = Script.get_stack_root() install_dir = stack_root + '/sqoop' download_url = config['configurations']['sqoop-env']['download_url'] filename = download_url.split('/')[-1] version_dir = filename.replace('.tar.gz', '').replace('.tgz', '') # Needed since this is an Atlas Hook service. cluster_name = config['clusterName'] ambari_server_hostname = config['ambariLevelParams']['ambari_server_host'] stack_name = default("/clusterLevelParams/stack_name", None) agent_stack_retry_on_unavailability = config['ambariLevelParams'][ 'agent_stack_retry_on_unavailability'] agent_stack_retry_count = expect("/ambariLevelParams/agent_stack_retry_count",