def test_normalized(self, log_error, log_info, warning_info, dir_mock): """ Test that the data dirs are normalized by removing leading and trailing whitespace, and case sensitive. """ params = StubParams() params.data_dir_mount_file = "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist" params.dfs_data_dir = "/grid/0/data , /grid/1/data ,/GRID/2/Data/" # Function under test mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, params.dfs_data_dir, params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] for (name, args, kwargs) in log_error.mock_calls: print args[0] log_info.assert_any_call("Forcefully creating directory: /grid/0/data") log_info.assert_any_call("Forcefully creating directory: /grid/1/data") log_info.assert_any_call( "Forcefully creating directory: /GRID/2/Data/") self.assertEquals(0, log_error.call_count)
def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point, mock_get_data_dir_to_mount_from_file, log_error, log_warning, log_info, dir_mock): """ Test when grid2 becomes unmounted """ mock_os_exists.return_value = True # Indicate that history file exists # Initially, all grids were mounted mock_get_data_dir_to_mount_from_file.return_value = {self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/dev2"} # Grid2 then becomes unmounted mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2 mock_os_isdir.side_effect = [False, False, False] + [True, True, True] # Function under test mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, self.params.dfs_data_dir, self.params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] error_logs = [] for (name, args, kwargs) in log_error.mock_calls: error_logs.append(args[0]) # this is a one-tuple error_msg = "".join(error_logs) self.assertEquals(1, log_error.call_count) self.assertTrue("Directory /grid/2/data became unmounted from /dev2 . Current mount point: / ." " Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of " "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist." in error_msg)
def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point, mock_get_data_dir_to_mount_from_file, log_error, log_info, dir_mock): """ Test when grid2 becomes remounted """ mock_os_exists.return_value = True # Indicate that history file exists # Initially, all grids were mounted mock_get_data_dir_to_mount_from_file.return_value = { self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/" } # Grid2 then becomes remounted mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/dev2"] * 2 mock_os_isdir.side_effect = [False, False, False] + [True, True, True] # Function under test mounted_dirs_helper.handle_mounted_dirs( fake_create_dir, self.params.data_dir_mount_file, self.params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] for (name, args, kwargs) in log_error.mock_calls: print args[0] self.assertEquals(0, log_error.call_count)
def setup_nodemanager(): import params try: Execute('umount /sys/fs/cgroup/cpu,cpuacct') Execute('mkdir -p /cgroups/cpu /cgroups/cpuacct') Execute('mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpu cgroup /cgroups/cpu') Execute('mount -t cgroup -o rw,nosuid,nodev,noexec,relatime,cpuacct cgroup /cgroups/cpuacct') Execute('chown -R yarn:hadoop /cgroups/cpu/yarn') except Exception as e: print str(e) # First start after enabling/disabling security if params.toggle_nm_security: Directory( params.nm_local_dirs_list + params.nm_log_dirs_list, action='delete') # If yarn.nodemanager.recovery.dir exists, remove this dir if params.yarn_nodemanager_recovery_dir: Directory( InlineTemplate(params.yarn_nodemanager_recovery_dir) .get_content(), action='delete') # Setting NM marker file if params.security_enabled: Directory(params.nm_security_marker_dir) File( params.nm_security_marker, content="Marker file to track first start after enabling/disabling security. " "During first start yarn local, log dirs are removed and recreated") elif not params.security_enabled: File(params.nm_security_marker, action="delete") if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs( create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) # create a history file used by handle_mounted_dirs File( params.nm_log_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_log_dir_to_mount_file_content) nm_local_dir_to_mount_file_content = handle_mounted_dirs( create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) File( params.nm_local_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_local_dir_to_mount_file_content)
def datanode(action=None): if action == "configure": import params Directory(params.dfs_domain_socket_dir, create_parents = True, mode=0751, owner=params.hdfs_user, group=params.user_group) # handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. data_dir_to_mount_file_content = handle_mounted_dirs(create_dirs, params.dfs_data_dirs, params.data_dir_mount_file, params) # create a history file used by handle_mounted_dirs File(params.data_dir_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=data_dir_to_mount_file_content ) elif action == "start" or action == "stop": import params service( action=action, name="datanode", user=params.hdfs_user, create_pid_dir=True, create_log_dir=True ) elif action == "status": import status_params check_process_status(status_params.datanode_pid_file)
def datanode(action=None): if action == "configure": import params Directory(params.dfs_domain_socket_dir, create_parents=True, mode=0751, owner=params.hdfs_user, group=params.user_group) # handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. data_dir_to_mount_file_content = handle_mounted_dirs( create_dirs, params.dfs_data_dirs, params.data_dir_mount_file, params) # create a history file used by handle_mounted_dirs File(params.data_dir_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=data_dir_to_mount_file_content) generate_logfeeder_input_config( 'hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) elif action == "start" or action == "stop": import params service(action=action, name="datanode", user=params.hdfs_user, create_pid_dir=True, create_log_dir=True) elif action == "status": import params PidFiles(params.possible_datanode_pid_files).check_status()
def setup_nodemanager(): import params # First start after enabling/disabling security if params.toggle_nm_security: Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, action='delete') # If yarn.nodemanager.recovery.dir exists, remove this dir if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate( params.yarn_nodemanager_recovery_dir).get_content(), action='delete') # Setting NM marker file if params.security_enabled: Directory(params.nm_security_marker_dir) File( params.nm_security_marker, content= "Marker file to track first start after enabling/disabling security. " "During first start yarn local, log dirs are removed and recreated" ) elif not params.security_enabled: File(params.nm_security_marker, action="delete") if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs( create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) # create a history file used by handle_mounted_dirs File(params.nm_log_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_log_dir_to_mount_file_content) nm_local_dir_to_mount_file_content = handle_mounted_dirs( create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) File(params.nm_local_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_local_dir_to_mount_file_content)
def test_normalized(self, log_error, log_info, warning_info, dir_mock): """ Test that the data dirs are normalized by removing leading and trailing whitespace, and case sensitive. """ params = StubParams() params.data_dir_mount_file = "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist" params.dfs_data_dir = "/grid/0/data , /grid/1/data ,/GRID/2/Data/" # Function under test mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, params.dfs_data_dir, params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] for (name, args, kwargs) in log_error.mock_calls: print args[0] log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /grid/0/data") log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /grid/1/data") log_info.assert_any_call("Forcefully ensuring existence and permissions of the directory: /GRID/2/Data/") self.assertEquals(0, log_error.call_count)
def test_grid_becomes_unmounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point, mock_get_data_dir_to_mount_from_file, log_error, log_info, dir_mock): """ Test when grid2 becomes unmounted """ mock_os_exists.return_value = True # Indicate that history file exists # Initially, all grids were mounted mock_get_data_dir_to_mount_from_file.return_value = { self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/dev2" } # Grid2 then becomes unmounted mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/"] * 2 mock_os_isdir.side_effect = [False, False, False] + [True, True, True] # Function under test mounted_dirs_helper.handle_mounted_dirs( fake_create_dir, self.params.dfs_data_dir, self.params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] error_logs = [] for (name, args, kwargs) in log_error.mock_calls: error_logs.append(args[0]) # this is a one-tuple error_msg = "".join(error_logs) self.assertEquals(1, log_error.call_count) self.assertTrue( "Directory /grid/2/data does not exist and became unmounted from /dev2" in error_msg)
def test_grid_becomes_remounted(self, mock_os_exists, mock_os_isdir, mock_get_mount_point, mock_get_data_dir_to_mount_from_file, log_error, log_warning, log_info, dir_mock): """ Test when grid2 becomes remounted """ mock_os_exists.return_value = True # Indicate that history file exists # Initially, all grids were mounted mock_get_data_dir_to_mount_from_file.return_value = {self.grid0: "/dev0", self.grid1: "/dev1", self.grid2: "/"} # Grid2 then becomes remounted mock_get_mount_point.side_effect = ["/dev0", "/dev1", "/dev2"] * 2 mock_os_isdir.side_effect = [False, False, False] + [True, True, True] # Function under test mounted_dirs_helper.handle_mounted_dirs(fake_create_dir, self.params.data_dir_mount_file, self.params.data_dir_mount_file, update_cache=False) for (name, args, kwargs) in log_info.mock_calls: print args[0] for (name, args, kwargs) in log_error.mock_calls: print args[0] self.assertEquals(0, log_error.call_count)
def yarn(name=None, config_dir=None): """ :param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client) :param config_dir: Which config directory to write configs to, which could be different during rolling upgrade. """ import params if config_dir is None: config_dir = params.hadoop_conf_dir if name == "historyserver": if params.yarn_log_aggregation_enabled: params.HdfsResource(params.yarn_nm_app_log_dir, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group, mode=0777, recursive_chmod=True) # create the /tmp folder with proper permissions if it doesn't exist yet if params.entity_file_history_directory.startswith('/tmp'): params.HdfsResource( params.hdfs_tmp_dir, action="create_on_execute", type="directory", owner=params.hdfs_user, mode=0777, ) params.HdfsResource(params.entity_file_history_directory, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group) params.HdfsResource("/mapred", type="directory", action="create_on_execute", owner=params.mapred_user) params.HdfsResource("/mapred/system", type="directory", action="create_on_execute", owner=params.hdfs_user) params.HdfsResource(params.mapreduce_jobhistory_done_dir, type="directory", action="create_on_execute", owner=params.mapred_user, group=params.user_group, change_permissions_for_parents=True, mode=0777) params.HdfsResource(None, action="execute") Directory( params.jhs_leveldb_state_store_dir, owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access="a", recursive_ownership=True, ) #<editor-fold desc="Node Manager Section"> if name == "nodemanager": # First start after enabling/disabling security if params.toggle_nm_security: Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, action='delete') # If yarn.nodemanager.recovery.dir exists, remove this dir if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate( params.yarn_nodemanager_recovery_dir).get_content(), action='delete') # Setting NM marker file if params.security_enabled: Directory(params.nm_security_marker_dir) File( params.nm_security_marker, content= "Marker file to track first start after enabling/disabling security. " "During first start yarn local, log dirs are removed and recreated" ) elif not params.security_enabled: File(params.nm_security_marker, action="delete") if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs( create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) # create a history file used by handle_mounted_dirs File(params.nm_log_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_log_dir_to_mount_file_content) nm_local_dir_to_mount_file_content = handle_mounted_dirs( create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) File(params.nm_local_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_local_dir_to_mount_file_content) #</editor-fold> if params.yarn_nodemanager_recovery_dir: Directory( InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, create_parents=True, mode=0755, cd_access='a', ) Directory( [params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [ params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir ], owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [params.yarn_log_dir_prefix], owner=params.yarn_user, group=params.user_group, create_parents=True, ignore_failures=True, cd_access='a', ) XmlConfig( "core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, # RU should rely on all available in <stack-root>/<version>/hadoop/conf if 'hdfs-site' in params.config['configurations']: XmlConfig( "hdfs-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) XmlConfig( "mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "yarn-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations']['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.yarn_user, group=params.user_group, mode=0644) if name == 'resourcemanager': Directory( params.rm_nodes_exclude_dir, mode=0755, create_parents=True, cd_access='a', ) File(params.rm_nodes_exclude_path, owner=params.yarn_user, group=params.user_group) File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group) if not is_empty( params.node_label_enable ) and params.node_label_enable or is_empty( params.node_label_enable) and params.node_labels_dir: params.HdfsResource(params.node_labels_dir, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") elif name == 'apptimelineserver': Directory( params.ats_leveldb_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) # if stack support application timeline-service state store property (timeline_state_store stack feature) if params.stack_supports_timeline_state_store: Directory( params.ats_leveldb_state_store_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) # app timeline server 1.5 directories if not is_empty(params.entity_groupfs_store_dir): parent_path = os.path.dirname(params.entity_groupfs_store_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_store_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_store_dir_mode) if not is_empty(params.entity_groupfs_active_dir): parent_path = os.path.dirname(params.entity_groupfs_active_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_active_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_active_dir_mode) params.HdfsResource(None, action="execute") File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2')) File(format("{limits_conf_dir}/mapreduce.conf"), mode=0644, content=Template('mapreduce.conf.j2')) File(os.path.join(config_dir, "yarn-env.sh"), owner=params.yarn_user, group=params.user_group, mode=0755, content=InlineTemplate(params.yarn_env_sh_template)) container_executor = format("{yarn_container_bin}/container-executor") File(container_executor, group=params.yarn_executor_container_group, mode=params.container_executor_mode) File(os.path.join(config_dir, "container-executor.cfg"), group=params.user_group, mode=0644, content=Template('container-executor.cfg.j2')) Directory(params.cgroups_dir, group=params.user_group, create_parents=True, mode=0755, cd_access="a") if params.security_enabled: tc_mode = 0644 tc_owner = "root" else: tc_mode = None tc_owner = params.hdfs_user File(os.path.join(config_dir, "mapred-env.sh"), owner=tc_owner, mode=0755, content=InlineTemplate(params.mapred_env_sh_template)) if params.security_enabled: File(os.path.join(params.hadoop_bin, "task-controller"), owner="root", group=params.mapred_tt_group, mode=06050) File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=tc_owner, mode=tc_mode, group=params.mapred_tt_group, content=Template("taskcontroller.cfg.j2")) else: File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=tc_owner, content=Template("taskcontroller.cfg.j2")) if "mapred-site" in params.config['configurations']: XmlConfig( "mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.mapred_user, group=params.user_group) if "capacity-scheduler" in params.config['configurations']: XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations'] ['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.hdfs_user, group=params.user_group) if "ssl-client" in params.config['configurations']: XmlConfig( "ssl-client.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) Directory( params.hadoop_conf_secure_dir, create_parents=True, owner='root', group=params.user_group, cd_access='a', ) XmlConfig( "ssl-client.xml", conf_dir=params.hadoop_conf_secure_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) if "ssl-server" in params.config['configurations']: XmlConfig( "ssl-server.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-server'], configuration_attributes=params.config['configuration_attributes'] ['ssl-server'], owner=params.hdfs_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')): File(os.path.join(config_dir, 'fair-scheduler.xml'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')): File(os.path.join(config_dir, 'ssl-client.xml.example'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group)
def yarn(name=None): import params if name == "historyserver": if params.yarn_log_aggregation_enabled: params.HdfsResource(params.yarn_nm_app_log_dir, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group, mode=0777, recursive_chmod=True) params.HdfsResource(params.entity_file_history_directory, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group) params.HdfsResource("/mapred", type="directory", action="create_on_execute", owner=params.mapred_user) params.HdfsResource("/mapred/system", type="directory", action="create_on_execute", owner=params.hdfs_user) params.HdfsResource(params.mapreduce_jobhistory_intermediate_done_dir, type="directory", action="create_on_execute", owner=params.mapred_user, group=params.user_group, mode=0777) params.HdfsResource(params.mapreduce_jobhistory_done_dir, type="directory", action="create_on_execute", owner=params.mapred_user, group=params.user_group, change_permissions_for_parents=True, mode=0777) params.HdfsResource(None, action="execute") Directory( params.jhs_leveldb_state_store_dir, owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access="a", ) Execute( ("chown", "-R", format("{mapred_user}:{user_group}"), params.jhs_leveldb_state_store_dir), sudo=True, ) if name == "nodemanager": # First start after enabling/disabling security if params.toggle_nm_security: Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, action='delete') # If yarn.nodemanager.recovery.dir exists, remove this dir if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate( params.yarn_nodemanager_recovery_dir).get_content(), action='delete') # Setting NM marker file if params.security_enabled: Directory(params.nm_security_marker_dir) File( params.nm_security_marker, content= "Marker file to track first start after enabling/disabling security. " "During first start yarn local, log dirs are removed and recreated" ) elif not params.security_enabled: File(params.nm_security_marker, action="delete") if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs( create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) # create a history file used by handle_mounted_dirs File(params.nm_log_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_log_dir_to_mount_file_content) nm_local_dir_to_mount_file_content = handle_mounted_dirs( create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) File(params.nm_local_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_local_dir_to_mount_file_content) if params.yarn_nodemanager_recovery_dir: Directory( InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, create_parents=True, mode=0755, cd_access='a', ) if params.security_enabled: smokeuser_directories = [ os.path.join(dir, 'usercache', params.smokeuser) for dir in params.nm_local_dirs.split(',') ] for directory in smokeuser_directories: Execute( ('chown', '-R', params.smokeuser, directory), only_if=format("test -d {directory}"), sudo=True, ) Directory( [params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [ params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir ], owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [params.yarn_log_dir_prefix], owner=params.yarn_user, create_parents=True, ignore_failures=True, cd_access='a', ) XmlConfig( "core-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, # RU should rely on all available in /usr/hdp/<version>/hadoop/conf XmlConfig( "hdfs-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) XmlConfig( "mapred-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "yarn-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "capacity-scheduler.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.yarn_user, group=params.user_group, mode=0644) if name == 'resourcemanager': File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group) if not is_empty( params.node_label_enable ) and params.node_label_enable or is_empty( params.node_label_enable) and params.node_labels_dir: params.HdfsResource(params.node_labels_dir, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") elif name == 'apptimelineserver': Directory( params.ats_leveldb_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) Directory( params.ats_leveldb_state_store_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) # app timeline server 1.5 directories if not is_empty(params.entity_groupfs_store_dir): parent_path = os.path.dirname(params.entity_groupfs_store_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_store_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_store_dir_mode) if not is_empty(params.entity_groupfs_active_dir): parent_path = os.path.dirname(params.entity_groupfs_active_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_active_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_active_dir_mode) params.HdfsResource(None, action="execute") File(params.rm_nodes_exclude_path, owner=params.yarn_user, group=params.user_group) File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2')) File(format("{limits_conf_dir}/mapreduce.conf"), mode=0644, content=Template('mapreduce.conf.j2')) File(format("{hadoop_conf_dir}/yarn-env.sh"), owner=params.yarn_user, group=params.user_group, mode=0755, content=InlineTemplate(params.yarn_env_sh_template)) container_executor = format("{yarn_container_bin}/container-executor") File(container_executor, group=params.yarn_executor_container_group, mode=params.container_executor_mode) File(format("{hadoop_conf_dir}/container-executor.cfg"), group=params.user_group, mode=0644, content=Template('container-executor.cfg.j2')) Directory(params.cgroups_dir, group=params.user_group, create_parents=True, mode=0755, cd_access="a") if params.security_enabled: tc_mode = 0644 tc_owner = "root" else: tc_mode = None tc_owner = params.hdfs_user File(format("{hadoop_conf_dir}/mapred-env.sh"), owner=tc_owner, mode=0755, content=InlineTemplate(params.mapred_env_sh_template)) if params.security_enabled: File(os.path.join(params.hadoop_bin, "task-controller"), owner="root", group=params.mapred_tt_group, mode=06050) File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), owner=tc_owner, mode=tc_mode, group=params.mapred_tt_group, content=Template("taskcontroller.cfg.j2")) else: File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'), owner=tc_owner, content=Template("taskcontroller.cfg.j2")) if "mapred-site" in params.config['configurations']: XmlConfig( "mapred-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.mapred_user, group=params.user_group) if "capacity-scheduler" in params.config['configurations']: XmlConfig( "capacity-scheduler.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations'] ['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.hdfs_user, group=params.user_group) if "ssl-client" in params.config['configurations']: XmlConfig( "ssl-client.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) Directory( params.hadoop_conf_secure_dir, create_parents=True, owner='root', group=params.user_group, cd_access='a', ) XmlConfig( "ssl-client.xml", conf_dir=params.hadoop_conf_secure_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) if "ssl-server" in params.config['configurations']: XmlConfig( "ssl-server.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['ssl-server'], configuration_attributes=params.config['configuration_attributes'] ['ssl-server'], owner=params.hdfs_user, group=params.user_group) if os.path.exists( os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')): File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'), owner=params.mapred_user, group=params.user_group) if os.path.exists( os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')): File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'), owner=params.mapred_user, group=params.user_group) if os.path.exists( os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')): File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group)