def link_config(old_conf, link_conf):
"""
Creates a config link following:
1. Checks if the old_conf location exists
2. If it does, check if it's a link already
3. Make a copy to /etc/[component]/conf.backup
4. Remove the old directory and create a symlink to link_conf
:old_conf: the old config directory, ie /etc/[component]/conf
:link_conf: the new target for the config directory, ie <stack-root>/current/[component-dir]/conf
"""
if os.path.islink(old_conf):
# if the link exists but is wrong, then change it
if os.path.realpath(old_conf) != link_conf:
Link(old_conf, to = link_conf)
else:
Logger.debug("Skipping {0}; it is already a link".format(old_conf))
return
if not os.path.exists(old_conf):
Logger.debug("Skipping {0}; it does not exist".format(old_conf))
return
old_parent = os.path.abspath(os.path.join(old_conf, os.pardir))
Logger.info("Linking {0} to {1}".format(old_conf, link_conf))
old_conf_copy = os.path.join(old_parent, "conf.backup")
if not os.path.exists(old_conf_copy):
Execute(("cp", "-R", "-p", old_conf, old_conf_copy), sudo=True, logoutput=True)
shutil.rmtree(old_conf, ignore_errors=True)
# link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf
Link(old_conf, to = link_conf)
def install_ranger(self):
import params
splice_lib_dir = "/var/lib/splicemachine"
ranger_home = format('{params.stack_root}/current/ranger-admin')
ranger_user = params.config['configurations']['ranger-env'][
'ranger_user']
ranger_plugins_dir = os.path.join(
ranger_home,
"ews/webapp/WEB-INF/classes/ranger-plugins/splicemachine")
Directory(ranger_plugins_dir,
owner=ranger_user,
group=ranger_user,
create_parents=False)
splice_ranger_jar = self.search_file(splice_lib_dir,
"splice_ranger_admin-hdp*.jar")
db_client_jar = self.search_file(splice_lib_dir, "db-client-*.jar")
Link(os.path.join(ranger_plugins_dir, splice_ranger_jar),
to=os.path.join(splice_lib_dir, splice_ranger_jar))
Link(os.path.join(ranger_plugins_dir, db_client_jar),
to=os.path.join(splice_lib_dir, db_client_jar))
hbase_user = params.config['configurations']['hbase-env']['hbase_user']
hdfs_audit_dir = params.config['configurations'][
'ranger-splicemachine-audit'][
'xasecure.audit.destination.hdfs.dir']
params.HdfsResource(hdfs_audit_dir,
type="directory",
action="create_on_execute",
owner=hbase_user)
def setup_atlas_falcon():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"falcon")
if os.path.exists(atlas_falcon_hook_dir):
Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file),
to=os.path.join(params.atlas_conf_dir,
params.atlas_conf_file))
Directory(params.falcon_webinf_lib,
owner=params.falcon_user,
create_parents=True)
src_files = os.listdir(atlas_falcon_hook_dir)
for file_name in src_files:
atlas_falcon_hook_file_name = os.path.join(
atlas_falcon_hook_dir, file_name)
falcon_lib_file_name = os.path.join(params.falcon_webinf_lib,
file_name)
if (os.path.isfile(atlas_falcon_hook_file_name)):
Link(falcon_lib_file_name, to=atlas_falcon_hook_file_name)
def select(stack_name, package, version, try_create=True, ignore_errors=False):
"""
Selects a config version for the specified package. If this detects that
the stack supports configuration versioning but /etc/<component>/conf is a
directory, then it will attempt to bootstrap the conf.backup directory and change
/etc/<component>/conf into a symlink.
:param stack_name: the name of the stack
:param package: the name of the package, as-used by <conf-selector-tool>
:param version: the version number to create
:param try_create: optional argument to attempt to create the directory before setting it
:param ignore_errors: optional argument to ignore any error and simply log a warning
"""
try:
# do nothing if the stack does not support versioned configurations
if not _valid(stack_name, package, version):
return
if try_create:
create(stack_name, package, version)
shell.checked_call(_get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)
# for consistency sake, we must ensure that the /etc/<component>/conf symlink exists and
# points to <stack-root>/current/<component>/conf - this is because some people still prefer to
# use /etc/<component>/conf even though <stack-root> is the "future"
package_dirs = get_package_dirs()
if package in package_dirs:
Logger.info("Ensuring that {0} has the correct symlink structure".format(package))
directory_list = package_dirs[package]
for directory_structure in directory_list:
conf_dir = directory_structure["conf_dir"]
current_dir = directory_structure["current_dir"]
# if /etc/<component>/conf is missing or is not a symlink
if not os.path.islink(conf_dir):
# if /etc/<component>/conf is not a link and it exists, convert it to a symlink
if os.path.exists(conf_dir):
parent_directory = os.path.dirname(conf_dir)
conf_backup_dir = os.path.join(parent_directory, "conf.backup")
# create conf.backup and copy files to it (if it doesn't exist)
Execute(("cp", "-R", "-p", conf_dir, conf_backup_dir),
not_if = format("test -e {conf_backup_dir}"), sudo = True)
# delete the old /etc/<component>/conf directory and link to the backup
Directory(conf_dir, action="delete")
Link(conf_dir, to = conf_backup_dir)
else:
# missing entirely
# /etc/<component>/conf -> <stack-root>/current/<component>/conf
Link(conf_dir, to = current_dir)
except Exception, exception:
if ignore_errors is True:
Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package,
str(exception)))
else:
raise
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env)
daemon_cmd = format('{knox_bin} start')
populate_topology = format(
'cd {knox_conf_dir}/topologies/; {sudo} ambari-python-wrap ./generate_template.py '
+ params.HAServers +
' ; {sudo} chmod 640 *.xml; {sudo} chown knox:knox *.xml')
no_op_test = format(
'ls {knox_pid_file} >/dev/null 2>&1 && ps -p `cat {knox_pid_file}` >/dev/null 2>&1'
)
setup_ranger_knox(upgrade_type=upgrade_type)
# Used to setup symlink, needed to update the knox managed symlink, in case of custom locations
if os.path.islink(params.knox_managed_pid_symlink):
Link(
params.knox_managed_pid_symlink,
to=params.knox_pid_dir,
)
if os.path.islink(params.knox_managed_logs_symlink):
Link(
params.knox_managed_logs_symlink,
to=params.knox_logs_dir,
)
update_knox_logfolder_permissions()
Execute(populate_topology)
Execute(daemon_cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test)
def setup_atlas_sqoop():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"sqoop")
if os.path.exists(atlas_sqoop_hook_dir):
Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
to=os.path.join(params.atlas_conf_dir,
params.atlas_conf_file))
src_files = os.listdir(atlas_sqoop_hook_dir)
for file_name in src_files:
atlas_sqoop_hook_file_name = os.path.join(
atlas_sqoop_hook_dir, file_name)
sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
if (os.path.isfile(atlas_sqoop_hook_file_name)):
Link(sqoop_lib_file_name, to=atlas_sqoop_hook_file_name)
def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir):
import params
backup_folder_path = None
backup_folder_suffix = "_tmp"
if kafka_ambari_managed_dir != kafka_managed_dir:
if os.path.exists(
kafka_managed_dir) and not os.path.islink(kafka_managed_dir):
# Backup existing data before delete if config is changed repeatedly to/from default location at any point in time time, as there may be relevant contents (historic logs)
backup_folder_path = backup_dir_contents(kafka_managed_dir,
backup_folder_suffix)
Directory(kafka_managed_dir, action="delete", create_parents=True)
elif os.path.islink(kafka_managed_dir) and os.path.realpath(
kafka_managed_dir) != kafka_ambari_managed_dir:
Link(kafka_managed_dir, action="delete")
if not os.path.islink(kafka_managed_dir):
Link(kafka_managed_dir, to=kafka_ambari_managed_dir)
elif os.path.islink(
kafka_managed_dir
): # If config is changed and coincides with the kafka managed dir, remove the symlink and physically create the folder
Link(kafka_managed_dir, action="delete")
Directory(
kafka_managed_dir,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
if backup_folder_path:
# Restore backed up files to current relevant dirs if needed - will be triggered only when changing to/from default path;
for file in os.listdir(backup_folder_path):
if os.path.isdir(os.path.join(backup_folder_path, file)):
Execute(('cp', '-r', os.path.join(backup_folder_path,
file), kafka_managed_dir),
sudo=True)
Execute(("chown", "-R", format("{kafka_user}:{user_group}"),
os.path.join(kafka_managed_dir, file)),
sudo=True)
else:
File(os.path.join(kafka_managed_dir, file),
owner=params.kafka_user,
content=StaticFile(os.path.join(backup_folder_path,
file)))
# Clean up backed up folder
Directory(backup_folder_path, action="delete", create_parents=True)
def install_snappy():
import params
Directory([params.so_target_dir_x86, params.so_target_dir_x64],
create_parents = True,
)
Link(params.so_target_x86,
to=params.so_src_x86,
)
Link(params.so_target_x64,
to=params.so_src_x64,
)
def install(self, env):
import params
presto_cli_path = params.presto_cli_path
Directory(presto_cli_path,
create_parents=True,
owner='root',
group='root')
Execute('wget --no-check-certificate {0} -O {1}'.format(
PRESTO_CLI_URL, presto_cli_path + '/presto-cli'))
Execute('chmod +x {0}'.format(presto_cli_path + '/presto-cli'))
Link('/usr/hdp/current/presto-client', to=params.presto_cli_path)
Link('/usr/bin/presto-cli',
to='/usr/hdp/current/presto-client/presto-cli')
def link_metrics_sink_jar(self):
# Add storm metrics reporter JAR to storm-ui-server classpath.
# Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
action="delete")
# On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar",
action="delete")
Execute(format(
"{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"
),
not_if=format(
"ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {metric_collector_sink_jar}"))
def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
"""
In HDP 2.3, 2.4, and 2.5.0.0, Sqoop and Storm still relied on the following method to setup Atlas hooks
because the RPM for Sqoop and Storm did not bring in any dependencies.
/usr/hdp/current/storm-*/libext/ should contain symlinks for every jar in /usr/hdp/current/atlas-server/hooks/storm/somejavafile.jar
/usr/hdp/current/sqoop-*/lib/ should contain symlinks for every jar in /usr/hdp/current/atlas-server/hooks/sqoop/somejavafile.jar
In HDP 2.5.x.y, we plan to have the Sqoop and Storm rpms have additional dependencies on some sqoop-atlas-hook and storm-atlas-hook
rpms, respectively, that will bring in the necessary jars and create the symlinks.
If atlas is present on this host, then link the jars from
{stack_root}/current/{hook_name}/lib/name_version.jar -> {jar_source_dir}/name_version.jar
@param hook_name: one of sqoop, storm
@param jar_source_dir: directory of where the symlinks need to be created from.
"""
import params
if has_atlas_in_cluster():
atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ \
else format("{stack_root}/current/atlas-server")
# Will only exist if this host contains Atlas Server
atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name)
if os.path.exists(atlas_hook_dir):
Logger.info(
"Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done."
% (jar_source_dir, atlas_hook_dir))
src_files = os.listdir(atlas_hook_dir)
for file_name in src_files:
atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name)
source_lib_file_name = os.path.join(jar_source_dir, file_name)
if os.path.isfile(atlas_hook_file_name):
Link(source_lib_file_name, to=atlas_hook_file_name)
def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
"""
@param hook_name: one of sqoop, storm, hive
@param jar_source_dir: directory of where the symlinks need to be created from.
"""
install_atlas_hook(hook_name)
stack_root = '/opt'
atlas_hook_dir = stack_root + '/atlas-' + hook_name + '-plugin'
if os.path.exists(atlas_hook_dir):
Logger.info(
"Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done."
% (jar_source_dir, atlas_hook_dir))
src_files = os.listdir(atlas_hook_dir)
for file_name in src_files:
atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name)
source_lib_file_name = os.path.join(jar_source_dir, file_name)
if os.path.isfile(atlas_hook_file_name):
Link(source_lib_file_name, to=atlas_hook_file_name)
else:
Logger.info("Atlas hook directory path {0} doesn't exist".format(
atlas_hook_dir))
def sqoop(type=None):
import params
Link(params.sqoop_lib + "/mysql-connector-java.jar",
to='/usr/share/java/mysql-connector-java.jar')
jdbc_connector()
Directory(params.sqoop_conf_dir,
owner=params.sqoop_user,
group=params.user_group,
create_parents=True)
configs = {}
configs.update(params.config['configurations']['sqoop-site'])
XmlConfig(
"sqoop-site.xml",
conf_dir=params.sqoop_conf_dir,
configurations=configs,
configuration_attributes=params.config['configuration_attributes']
['sqoop-site'],
owner=params.sqoop_user,
group=params.user_group)
setup_atlas_sqoop()
File(format("{sqoop_conf_dir}/sqoop-env.sh"),
owner=params.sqoop_user,
group=params.user_group,
content=InlineTemplate(params.sqoop_env_sh_template))
update_config_permissions(
["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"])
pass
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info("Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info("Unrecognized stack id {0}, cannot create config links".format(stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(stack_version))
return
# After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink
if self.stack_name.upper() == "HDF" \
and not os.path.exists("/usr/bin/conf-select") and os.path.exists("/usr/bin/hdfconf-select"):
Link("/usr/bin/conf-select", to = "/usr/bin/hdfconf-select")
for package_name, directories in conf_select.get_package_dirs().iteritems():
conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
Logger.info("The current cluster stack of {0} does not require backing up configurations; "
"only {1} versioned config directories will be created.".format(stack_version, conf_selector_name))
# only link configs for all known packages
conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
def setup_atlas_storm():
import params
if params.has_atlas:
if not params.host_sys_prepped:
Package(
params.atlas_ubuntu_plugin_package
if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
PropertiesFile(format('{conf_dir}/{atlas_conf_file}'),
properties=params.atlas_props,
owner=params.storm_user,
group=params.user_group,
mode=0644)
atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"storm")
if os.path.exists(atlas_storm_hook_dir):
storm_extlib_dir = os.path.join(params.storm_component_home_dir,
"extlib")
if os.path.exists(storm_extlib_dir):
src_files = os.listdir(atlas_storm_hook_dir)
for file_name in src_files:
atlas_storm_hook_file_name = os.path.join(
atlas_storm_hook_dir, file_name)
storm_lib_file_name = os.path.join(storm_extlib_dir,
file_name)
if (os.path.isfile(atlas_storm_hook_file_name)):
Link(storm_lib_file_name,
to=atlas_storm_hook_file_name)
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env)
daemon_cmd = format('{knox_bin} start')
no_op_test = format(
'ls {knox_pid_file} >/dev/null 2>&1 && ps -p `cat {knox_pid_file}` >/dev/null 2>&1'
)
setup_ranger_knox(upgrade_type=upgrade_type)
# Used to setup symlink, needed to update the knox managed symlink, in case of custom locations
if os.path.islink(params.knox_managed_pid_symlink):
Link(
params.knox_managed_pid_symlink,
to=params.knox_pid_dir,
)
update_knox_logfolder_permissions()
try:
Execute(daemon_cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=no_op_test)
except:
show_logs(params.knox_logs_dir, params.knox_user)
raise
def sqoop(type=None):
import params
Link(params.sqoop_lib + "/mysql-connector-java.jar",
to='/usr/share/java/mysql-connector-java.jar')
jdbc_connector()
Directory(params.sqoop_conf_dir,
owner=params.sqoop_user,
group=params.user_group,
create_parents=True)
configs = {}
configs.update(params.config['configurations']['sqoop-site'])
XmlConfig(
"sqoop-site.xml",
conf_dir=params.sqoop_conf_dir,
configurations=configs,
configuration_attributes=params.config['configuration_attributes']
['sqoop-site'],
owner=params.sqoop_user,
group=params.user_group)
if params.has_atlas:
atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook",
"sqoop")
if os.path.exists(atlas_sqoop_hook_dir):
Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
to=os.path.join(params.atlas_conf_dir,
params.atlas_conf_file))
src_files = os.listdir(atlas_sqoop_hook_dir)
for file_name in src_files:
atlas_sqoop_hook_file_name = os.path.join(
atlas_sqoop_hook_dir, file_name)
sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
if (os.path.isfile(atlas_sqoop_hook_file_name)):
Link(sqoop_lib_file_name, to=atlas_sqoop_hook_file_name)
File(format("{sqoop_conf_dir}/sqoop-env.sh"),
owner=params.sqoop_user,
group=params.user_group,
content=InlineTemplate(params.sqoop_env_sh_template))
update_config_permissions(
["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"])
pass
def test_action_delete(self, exists_mock, unlink_mock):
exists_mock.return_value = True
with Environment('/') as env:
Link("/some_path",
action = "delete"
)
unlink_mock.assert_called_with("/some_path")
def test_action_create_symlink_clean_create(self, symlink_mock,
lexists_mock):
lexists_mock.return_value = False
with Environment('/') as env:
Link("/some_path", to="/a/b/link_to_path")
symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
def test_action_create_hardlink_clean_create(self, link_mock, lexists_mock,
exists_mock, isdir_mock):
lexists_mock.return_value = False
exists_mock.return_value = True
isdir_mock.return_value = False
with Environment('/') as env:
Link("/some_path", hard=True, to="/a/b/link_to_path")
link_mock.assert_called_with("/a/b/link_to_path", "/some_path")
def test_action_create_relink(self, symlink_mock, unlink_mock, islink_mock,
lexists_mock, realmock):
lexists_mock.return_value = True
realmock.return_value = "/old_to_link_path"
islink_mock.return_value = True
with Environment('/') as env:
Link("/some_path", to="/a/b/link_to_path")
unlink_mock.assert_called_with("/some_path")
symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
def install(self, env):
from params import presto_server_path
presto_etc_path = presto_server_path + '/etc'
presto_catalog_path = presto_etc_path + '/catalog'
Execute('wget --no-check-certificate {0} -O /tmp/{1}'.format(PRESTO_TAR_URL, 'presto-server.tar.gz'))
Directory([presto_server_path, presto_etc_path, presto_catalog_path], create_parents=True, owner='root', group='root')
Execute('tar xf {0} -C {1} --strip-components 1'.format('/tmp/presto-server.tar.gz', presto_server_path))
Link('/usr/hdp/current/presto-server', to=presto_server_path)
self.install_packages(env)
self.configure(env)
def test_action_create_hardlink_target_doesnt_exist(
self, lexists_mock, exists_mock):
lexists_mock.return_value = False
exists_mock.return_value = False
with Environment('/') as env:
try:
Link("/some_path", hard=True, to="/a/b/link_to_path")
self.fail("Must fail when target directory do doenst exist")
except Fail as e:
self.assertEqual(
'Failed to apply Link[\'/some_path\'], linking to nonexistent location /a/b/link_to_path',
str(e))
def link_metrics_sink_jar(self):
import params
# Add storm metrics reporter JAR to storm-ui-server classpath.
# Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
action="delete")
# On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar",
action="delete")
if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES,
params.version_for_stack_feature_checks):
sink_jar = params.metric_collector_sink_jar
else:
sink_jar = params.metric_collector_legacy_sink_jar
Execute(format(
"{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"
),
not_if=format(
"ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {sink_jar}"))
def test_action_create_hardlink_target_is_dir(self, lexists_mock,
exists_mock, isdir_mock):
lexists_mock.return_value = False
exists_mock.return_value = True
isdir_mock = True
with Environment('/') as env:
try:
Link("/some_path", hard=True, to="/a/b/link_to_path")
self.fail("Must fail when hardlinking to directory")
except Fail as e:
self.assertEqual(
'Failed to apply Link[\'/some_path\'], cannot create hard link to a directory (/a/b/link_to_path)',
str(e))
def registry(env, upgrade_type=None):
import params
ensure_base_directories()
File(format("{conf_dir}/registry-env.sh"),
owner=params.registry_user,
content=InlineTemplate(params.registry_env_sh_template))
File(params.bootstrap_storage_command,
owner=params.registry_user,
content=InlineTemplate(params.bootstrap_sh_template),
mode=0755)
if params.security_enabled:
if params.registry_jaas_conf_template:
File(format("{conf_dir}/registry_jaas.conf"),
owner=params.registry_user,
content=InlineTemplate(params.registry_jaas_conf_template))
else:
TemplateConfig(format("{conf_dir}/registry_jaas.conf"),
owner=params.registry_user)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
Directory(
[params.jar_storage],
owner=params.registry_user,
group=params.user_group,
create_parents=True,
cd_access="a",
mode=0755,
)
File(os.path.join(params.limits_conf_dir, 'registry.conf'),
owner='root',
group='root',
mode=0644,
content=Template("registry.conf.j2"))
File(format("{conf_dir}/registry.yaml"),
content=Template("registry.yaml.j2"),
owner=params.registry_user,
group=params.user_group,
mode=0644)
if not os.path.islink(params.registry_managed_log_dir):
Link(params.registry_managed_log_dir, to=params.registry_log_dir)
def test_action_create_failed_due_to_file_exists(self,
lexists_mock, realmock):
lexists_mock.side_effect = [True, False]
realmock.return_value = "/old_to_link_path"
with Environment('/') as env:
try:
Link("/some_path",
to = "/a/b/link_to_path"
)
self.fail("Must fail when directory or file with name /some_path exist")
except Fail as e:
self.assertEqual("Link['/some_path'] trying to create a symlink with the same name as an existing file or directory",
str(e))
def sqoop(type=None):
import params
Link(params.sqoop_lib + "/mysql-connector-java.jar",
to='/usr/share/java/mysql-connector-java.jar')
Directory(params.sqoop_conf_dir,
owner=params.sqoop_user,
group=params.user_group,
recursive=True)
File(format("{sqoop_conf_dir}/sqoop-env.sh"),
owner=params.sqoop_user,
group=params.user_group,
content=InlineTemplate(params.sqoop_env_sh_template))
update_config_permissions(
["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"])
pass
def select(stack_name, package, version, try_create=True, ignore_errors=False):
"""
Selects a config version for the specified package.
:param stack_name: the name of the stack
:param package: the name of the package, as-used by conf-select
:param version: the version number to create
:param try_create: optional argument to attempt to create the directory before setting it
:param ignore_errors: optional argument to ignore any error and simply log a warning
"""
try:
if not _valid(stack_name, package, version):
return
if try_create:
create(stack_name, package, version)
shell.checked_call(get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)
# for consistency sake, we must ensure that the /etc/<component>/conf symlink exists and
# points to /usr/hdp/current/<component>/conf - this is because some people still prefer to
# use /etc/<component>/conf even though /usr/hdp is the "future"
if package in PACKAGE_DIRS:
Logger.info("Ensuring that {0} has the correct symlink structure".format(package))
directory_list = PACKAGE_DIRS[package]
for directory_structure in directory_list:
conf_dir = directory_structure["conf_dir"]
current_dir = directory_structure["current_dir"]
# if /etc/<component>/conf is not a symlink, we need to change it
if not os.path.islink(conf_dir):
# if it exists, try to back it up
if os.path.exists(conf_dir):
parent_directory = os.path.dirname(conf_dir)
conf_install_dir = os.path.join(parent_directory, "conf.backup")
Execute(("cp", "-R", "-p", conf_dir, conf_install_dir),
not_if = format("test -e {conf_install_dir}"), sudo = True)
Directory(conf_dir, action="delete")
Link(conf_dir, to = current_dir)
except Exception, exception:
if ignore_errors is True:
Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package,
str(exception)))
else:
raise
def _relink_configurations_with_conf_select(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info(
"Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info(
"Unrecognized stack id {0}, cannot create config links".format(
stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(
stack_version))
return
# After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink
if self.stack_name.upper() == "HDF" \
and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"):
Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select")
restricted_packages = conf_select.get_restricted_packages()
if 0 == len(restricted_packages):
Logger.info(
"There are no restricted conf-select packages for this installation"
)
else:
Logger.info("Restricting conf-select packages to {0}".format(
restricted_packages))
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
if 0 == len(restricted_packages
) or package_name in restricted_packages:
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
