def test_normalized(self, log_error, log_info, warning_info, dir_mock):
"""
Test that the data dirs are normalized by removing leading and trailing whitespace, and case sensitive.
"""
params = StubParams()
params.data_dir_mount_file = "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist"
params.dfs_data_dir = "/grid/0/data , /grid/1/data ,/GRID/2/Data/"
# Function under test
mounted_dirs_helper.handle_mounted_dirs(fake_create_dir,
params.dfs_data_dir,
params.data_dir_mount_file,
update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
for (name, args, kwargs) in log_error.mock_calls:
print args[0]
log_info.assert_any_call("Forcefully creating directory: /grid/0/data")
log_info.assert_any_call("Forcefully creating directory: /grid/1/data")
log_info.assert_any_call(
"Forcefully creating directory: /GRID/2/Data/")
self.assertEquals(0, log_error.call_count)
def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_get_data_dir_to_mount_from_file, log_error, log_warning, log_info, dir_mock):
"""
Test when grid2 becomes unmounted
"""
mock_os_exists.return_value = True # Indicate that history file exists
# Initially, all grids were mounted
mock_get_data_dir_to_mount_from_file.return_value = {self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/dev2"}
# Grid2 then becomes unmounted
mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2
mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
# Function under test
mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, self.params.dfs_data_dir, self.params.data_dir_mount_file, update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
error_logs = []
for (name, args, kwargs) in log_error.mock_calls:
error_logs.append(args[0]) # this is a one-tuple
error_msg = "".join(error_logs)
self.assertEquals(1, log_error.call_count)
self.assertTrue("Directory /grid/2/data became unmounted from /dev2 . Current mount point: / ."
" Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of "
"/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist." in error_msg)
def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir,
mock_get_mount_point,
mock_get_data_dir_to_mount_from_file,
log_error, log_info, dir_mock):
"""
Test when grid2 becomes remounted
"""
mock_os_exists.return_value = True # Indicate that history file exists
# Initially, all grids were mounted
mock_get_data_dir_to_mount_from_file.return_value = {
self.grid0: "/dev0",
self.grid1: "/dev1",
self.grid2: "/"
}
# Grid2 then becomes remounted
mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/dev2"] * 2
mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
# Function under test
mounted_dirs_helper.handle_mounted_dirs(
fake_create_dir,
self.params.data_dir_mount_file,
self.params.data_dir_mount_file,
update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
for (name, args, kwargs) in log_error.mock_calls:
print args[0]
self.assertEquals(0, log_error.call_count)
def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_get_data_dir_to_mount_from_file, log_error, log_warning, log_info, dir_mock):
"""
Test when grid2 becomes unmounted
"""
mock_os_exists.return_value = True # Indicate that history file exists
# Initially, all grids were mounted
mock_get_data_dir_to_mount_from_file.return_value = {self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/dev2"}
# Grid2 then becomes unmounted
mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2
mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
# Function under test
mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, self.params.dfs_data_dir, self.params.data_dir_mount_file, update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
error_logs = []
for (name, args, kwargs) in log_error.mock_calls:
error_logs.append(args[0]) # this is a one-tuple
error_msg = "".join(error_logs)
self.assertEquals(1, log_error.call_count)
self.assertTrue("Directory /grid/2/data became unmounted from /dev2 . Current mount point: / ."
" Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of "
"/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist." in error_msg)
def setup_nodemanager():
import params
try:
Execute('umount /sys/fs/cgroup/cpu,cpuacct')
Execute('mkdir -p /cgroups/cpu /cgroups/cpuacct')
Execute('mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpu cgroup /cgroups/cpu')
Execute('mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpuacct cgroup /cgroups/cpuacct')
Execute('chown -R yarn:hadoop /cgroups/cpu/yarn')
except Exception as e:
print str(e)
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(
params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir)
.get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content="Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated")
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(
params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(
params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
def datanode(action=None):
if action == "configure":
import params
Directory(params.dfs_domain_socket_dir,
create_parents = True,
mode=0751,
owner=params.hdfs_user,
group=params.user_group)
# handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
data_dir_to_mount_file_content = handle_mounted_dirs(create_dirs, params.dfs_data_dirs, params.data_dir_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.data_dir_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=data_dir_to_mount_file_content
)
elif action == "start" or action == "stop":
import params
service(
action=action, name="datanode",
user=params.hdfs_user,
create_pid_dir=True,
create_log_dir=True
)
elif action == "status":
import status_params
check_process_status(status_params.datanode_pid_file)
def datanode(action=None):
if action == "configure":
import params
Directory(params.dfs_domain_socket_dir,
create_parents=True,
mode=0751,
owner=params.hdfs_user,
group=params.user_group)
# handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
data_dir_to_mount_file_content = handle_mounted_dirs(
create_dirs, params.dfs_data_dirs, params.data_dir_mount_file,
params)
# create a history file used by handle_mounted_dirs
File(params.data_dir_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=data_dir_to_mount_file_content)
generate_logfeeder_input_config(
'hdfs',
Template("input.config-hdfs.json.j2", extra_imports=[default]))
elif action == "start" or action == "stop":
import params
service(action=action,
name="datanode",
user=params.hdfs_user,
create_pid_dir=True,
create_log_dir=True)
elif action == "status":
import params
PidFiles(params.possible_datanode_pid_files).check_status()
def setup_nodemanager():
import params
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(
params.yarn_nodemanager_recovery_dir).get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content=
"Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated"
)
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
def test_normalized(self, log_error, log_info, warning_info, dir_mock):
"""
Test that the data dirs are normalized by removing leading and trailing whitespace, and case sensitive.
"""
params = StubParams()
params.data_dir_mount_file = "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist"
params.dfs_data_dir = "/grid/0/data , /grid/1/data ,/GRID/2/Data/"
# Function under test
mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, params.dfs_data_dir, params.data_dir_mount_file, update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
for (name, args, kwargs) in log_error.mock_calls:
print args[0]
log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /grid/0/data")
log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /grid/1/data")
log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /GRID/2/Data/")
self.assertEquals(0, log_error.call_count)
def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir,
mock_get_mount_point,
mock_get_data_dir_to_mount_from_file,
log_error, log_info, dir_mock):
"""
Test when grid2 becomes unmounted
"""
mock_os_exists.return_value = True # Indicate that history file exists
# Initially, all grids were mounted
mock_get_data_dir_to_mount_from_file.return_value = {
self.grid0: "/dev0",
self.grid1: "/dev1",
self.grid2: "/dev2"
}
# Grid2 then becomes unmounted
mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2
mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
# Function under test
mounted_dirs_helper.handle_mounted_dirs(
fake_create_dir,
self.params.dfs_data_dir,
self.params.data_dir_mount_file,
update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
error_logs = []
for (name, args, kwargs) in log_error.mock_calls:
error_logs.append(args[0]) # this is a one-tuple
error_msg = "".join(error_logs)
self.assertEquals(1, log_error.call_count)
self.assertTrue(
"Directory /grid/2/data does not exist and became unmounted from /dev2"
in error_msg)
def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point,
mock_get_data_dir_to_mount_from_file, log_error, log_warning, log_info, dir_mock):
"""
Test when grid2 becomes remounted
"""
mock_os_exists.return_value = True # Indicate that history file exists
# Initially, all grids were mounted
mock_get_data_dir_to_mount_from_file.return_value = {self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/"}
# Grid2 then becomes remounted
mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/dev2"] * 2
mock_os_isdir.side_effect = [False, False, False] + [True, True, True]
# Function under test
mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, self.params.data_dir_mount_file, self.params.data_dir_mount_file, update_cache=False)
for (name, args, kwargs) in log_info.mock_calls:
print args[0]
for (name, args, kwargs) in log_error.mock_calls:
print args[0]
self.assertEquals(0, log_error.call_count)
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
if config_dir is None:
config_dir = params.hadoop_conf_dir
if name == "historyserver":
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=0777,
recursive_chmod=True)
# create the /tmp folder with proper permissions if it doesn't exist yet
if params.entity_file_history_directory.startswith('/tmp'):
params.HdfsResource(
params.hdfs_tmp_dir,
action="create_on_execute",
type="directory",
owner=params.hdfs_user,
mode=0777,
)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
recursive_ownership=True,
)
#<editor-fold desc="Node Manager Section">
if name == "nodemanager":
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(
params.yarn_nodemanager_recovery_dir).get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content=
"Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated"
)
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
#</editor-fold>
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access='a',
)
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"yarn-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
if name == 'resourcemanager':
Directory(
params.rm_nodes_exclude_dir,
mode=0755,
create_parents=True,
cd_access='a',
)
File(params.rm_nodes_exclude_path,
owner=params.yarn_user,
group=params.user_group)
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group)
if not is_empty(
params.node_label_enable
) and params.node_label_enable or is_empty(
params.node_label_enable) and params.node_labels_dir:
params.HdfsResource(params.node_labels_dir,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
elif name == 'apptimelineserver':
Directory(
params.ats_leveldb_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# if stack support application timeline-service state store property (timeline_state_store stack feature)
if params.stack_supports_timeline_state_store:
Directory(
params.ats_leveldb_state_store_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# app timeline server 1.5 directories
if not is_empty(params.entity_groupfs_store_dir):
parent_path = os.path.dirname(params.entity_groupfs_store_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_store_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_store_dir_mode)
if not is_empty(params.entity_groupfs_active_dir):
parent_path = os.path.dirname(params.entity_groupfs_active_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_active_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_active_dir_mode)
params.HdfsResource(None, action="execute")
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
container_executor = format("{yarn_container_bin}/container-executor")
File(container_executor,
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0644,
content=Template('container-executor.cfg.j2'))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
if params.security_enabled:
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
File(os.path.join(config_dir, "mapred-env.sh"),
owner=tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
mode=tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
content=Template("taskcontroller.cfg.j2"))
if "mapred-site" in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
if "capacity-scheduler" in params.config['configurations']:
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']
['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')):
File(os.path.join(config_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')):
File(os.path.join(config_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
def yarn(name=None):
import params
if name == "historyserver":
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=0777,
recursive_chmod=True)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_intermediate_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
mode=0777)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
Execute(
("chown", "-R", format("{mapred_user}:{user_group}"),
params.jhs_leveldb_state_store_dir),
sudo=True,
)
if name == "nodemanager":
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(
params.yarn_nodemanager_recovery_dir).get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content=
"Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated"
)
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
if params.security_enabled:
smokeuser_directories = [
os.path.join(dir, 'usercache', params.smokeuser)
for dir in params.nm_local_dirs.split(',')
]
for directory in smokeuser_directories:
Execute(
('chown', '-R', params.smokeuser, directory),
only_if=format("test -d {directory}"),
sudo=True,
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
create_parents=True,
ignore_failures=True,
cd_access='a',
)
XmlConfig(
"core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in /usr/hdp/<version>/hadoop/conf
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig(
"mapred-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"yarn-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
if name == 'resourcemanager':
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group)
if not is_empty(
params.node_label_enable
) and params.node_label_enable or is_empty(
params.node_label_enable) and params.node_labels_dir:
params.HdfsResource(params.node_labels_dir,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
elif name == 'apptimelineserver':
Directory(
params.ats_leveldb_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
Directory(
params.ats_leveldb_state_store_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# app timeline server 1.5 directories
if not is_empty(params.entity_groupfs_store_dir):
parent_path = os.path.dirname(params.entity_groupfs_store_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_store_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_store_dir_mode)
if not is_empty(params.entity_groupfs_active_dir):
parent_path = os.path.dirname(params.entity_groupfs_active_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_active_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_active_dir_mode)
params.HdfsResource(None, action="execute")
File(params.rm_nodes_exclude_path,
owner=params.yarn_user,
group=params.user_group)
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(format("{hadoop_conf_dir}/yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
container_executor = format("{yarn_container_bin}/container-executor")
File(container_executor,
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(format("{hadoop_conf_dir}/container-executor.cfg"),
group=params.user_group,
mode=0644,
content=Template('container-executor.cfg.j2'))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
if params.security_enabled:
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
File(format("{hadoop_conf_dir}/mapred-env.sh"),
owner=tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
owner=tc_owner,
mode=tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
else:
File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
owner=tc_owner,
content=Template("taskcontroller.cfg.j2"))
if "mapred-site" in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
if "capacity-scheduler" in params.config['configurations']:
XmlConfig(
"capacity-scheduler.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']
['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(
os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')):
File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(
os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')):
File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(
os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')):
File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
