def test_at_drive(self, mounted_mock):
"""
Testing when the path is mounted on a virtual file system not at the root.
"""
mounted_mock.return_value = self._get_mount(self.MOUNT_TYPE.MULT_DRIVE_DISTINCT)
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data/1")
self.assertEqual(mount_point, "/hadoop/hdfs/data/1")
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data/2")
self.assertEqual(mount_point, "/hadoop/hdfs/data/2")
def test_invalid(self):
"""
Testing when parameters are invalid or missing.
"""
mount_point = file_system.get_mount_point_for_dir(None)
self.assertEqual(mount_point, None)
mount_point = file_system.get_mount_point_for_dir("")
self.assertEqual(mount_point, None)
mount_point = file_system.get_mount_point_for_dir(" ")
self.assertEqual(mount_point, None)
def test_invalid(self, log_error, log_info):
"""
Testing when parameters are invalid or missing.
"""
mount_point = file_system.get_mount_point_for_dir(None)
self.assertEqual(mount_point, None)
mount_point = file_system.get_mount_point_for_dir("")
self.assertEqual(mount_point, None)
mount_point = file_system.get_mount_point_for_dir(" ")
self.assertEqual(mount_point, None)
def test_at_drive(self, mounted_mock, log_error, log_info):
"""
Testing when the path is mounted on a virtual file system not at the root.
"""
mounted_mock.return_value = self._get_mount(
self.MOUNT_TYPE.MULT_DRIVE_DISTINCT)
mount_point = file_system.get_mount_point_for_dir(
"/hadoop/hdfs/data/1")
self.assertEqual(mount_point, "/hadoop/hdfs/data/1")
mount_point = file_system.get_mount_point_for_dir(
"/hadoop/hdfs/data/2")
self.assertEqual(mount_point, "/hadoop/hdfs/data/2")
def test_at_root(self, mounted_mock):
"""
Testing when the path is mounted on the root.
"""
mounted_mock.return_value = self._get_mount(self.MOUNT_TYPE.SINGLE_ROOT)
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data")
self.assertEqual(mount_point, "/")
def test_at_root(self, mounted_mock, log_error, log_info):
"""
Testing when the path is mounted on the root.
"""
mounted_mock.return_value = self._get_mount(
self.MOUNT_TYPE.SINGLE_ROOT)
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data")
self.assertEqual(mount_point, "/")
def test_get_mount_point_for_dir_with_mounsts(self, mounted_mock,
log_error, log_info):
"""
Testing get_mount_point_for_dir when mounsts are provided.
"""
mounted_mock.return_value = self._get_mount(
self.MOUNT_TYPE.SAME_PREFIX_MOUNTS)
# refresh cached mounts
file_system.get_and_cache_mount_points(True)
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1")
self.assertEqual(mount_point, "/hadoop/hdfs/data1")
# Should use provided mounts, not fetch via get_and_cache_mount_points
mount_point = file_system.get_mount_point_for_dir(
"/hadoop/hdfs/data1", ["/", "/hadoop"])
self.assertEqual(mount_point, "/hadoop")
def test_same_prefix(self, mounted_mock, log_error, log_info):
"""
Testing when two mount points have the same prefix.
"""
mounted_mock.return_value = self._get_mount(
self.MOUNT_TYPE.SAME_PREFIX_MOUNTS)
# refresh cached mounts
file_system.get_and_cache_mount_points(True)
mount_point = file_system.get_mount_point_for_dir("/hadoop/hdfs/data1")
self.assertEqual(mount_point, "/hadoop/hdfs/data1")
def test_one_segment_mount(self, mounted_mock, log_error, log_info):
"""
Testing when the path has one segment.
"""
mounted_mock.return_value = self._get_mount(
self.MOUNT_TYPE.ONE_SEGMENT_MOUNT)
# refresh cached mounts
file_system.get_and_cache_mount_points(True)
mount_point = file_system.get_mount_point_for_dir(
"/hadoop/hdfs/data/1")
self.assertEqual(mount_point, "/hadoop")
def get_mounts_with_multiple_data_dirs(mount_points, dirs):
"""
Returns a list with (mount, dir_list) for mounts with multiple dirs.
Currently is used in the stack_advisor.
"""
mount_dirs = defaultdict(list)
for dir in [raw_dir.strip() for raw_dir in dirs.split(",")]:
mount_point = get_mount_point_for_dir(dir, mount_points)
mount_dirs[mount_point].append(dir)
partition_mounts_list = []
for mount_point, dir_list in mount_dirs.iteritems():
if len(dir_list) > 1:
partition_mounts_list.append((mount_point, dir_list))
return partition_mounts_list
def get_mounts_with_multiple_data_dirs(mount_points, dirs):
"""
Returns a list with (mount, dir_list) for mounts with multiple dirs.
Currently is used in the stack_advisor.
"""
mount_dirs = defaultdict(list)
for dir in [raw_dir.strip() for raw_dir in dirs.split(",")]:
mount_point = get_mount_point_for_dir(dir, mount_points)
mount_dirs[mount_point].append(dir)
partition_mounts_list = []
for mount_point, dir_list in mount_dirs.iteritems():
if len(dir_list) > 1:
partition_mounts_list.append((mount_point, dir_list))
return partition_mounts_list
def handle_mounted_dirs(func,
dirs_string,
history_filename,
update_cache=True):
"""
This function determine which dir paths can be created.
There are 2 uses cases:
1. Customers that have many dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all dirs are mounted on the root.
The goal is to avoid forcefully creating a dir when a user's drive fails. In this scenario, the
mount point for a dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the history_file parameter to parse a file that contains
a mapping from a dir, and its last known mount point.
After determining which dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(dir)
:param update_cache: Bool indicating whether to update the global cache of mount points
:return: Returns a history_filename content
"""
Directory(
os.path.dirname(history_filename),
create_parents=True,
mode=0755,
)
# Get the dirs that Ambari knows about and their last known mount point
prev_dir_to_mount_point = get_dir_to_mount_from_file(history_filename)
# Dictionary from dir to the mount point that will be written to the history file.
# If a dir becomes unmounted, we should still keep its original value.
# If a dir was previously on / and is now mounted on a drive, we should store that too.
dir_to_mount_point = prev_dir_to_mount_point.copy()
# This should typically be False for customers, but True the first time.
allowed_to_create_any_dir = False
if history_filename is None:
allowed_to_create_any_dir = True
Logger.warning(
"handle_mounted_dirs is allowed to create any directory since history_file.file property is null."
)
else:
if not os.path.exists(history_filename):
allowed_to_create_any_dir = True
Logger.warning(
"handle_mounted_dirs is allowed to create any directory since history_file property has file %s and it does not exist."
% history_filename)
valid_dirs = [] # dirs that have been normalized
error_messages = [] # list of error messages to report at the end
dirs_unmounted = set() # set of dirs that have become unmounted
dirs_string = ",".join([
re.sub(r'^\[.+\]', '', dfs_dir.strip())
for dfs_dir in dirs_string.split(",")
])
for dir in dirs_string.split(","):
if dir is None or dir.strip() == "":
continue
dir = dir.strip()
valid_dirs.append(dir)
if not os.path.isdir(dir):
may_create_this_dir = allowed_to_create_any_dir
last_mount_point_for_dir = None
# Determine if should be allowed to create the dir directory.
# Either first time, became unmounted, or was just mounted on a drive
if not may_create_this_dir:
last_mount_point_for_dir = prev_dir_to_mount_point[
dir] if dir in prev_dir_to_mount_point else None
if last_mount_point_for_dir is None:
# Couldn't retrieve any information about where this dir used to be mounted, so allow creating the directory to be safe.
may_create_this_dir = True
else:
curr_mount_point = get_mount_point_for_dir(dir)
# This means that create_this_dir will stay false if the directory became unmounted.
# In other words, allow creating if it was already on /, or it's currently not on /
if last_mount_point_for_dir == "/" or (
curr_mount_point is not None
and curr_mount_point != "/"):
may_create_this_dir = True
if may_create_this_dir:
Logger.info("Forcefully creating directory: {0}".format(dir))
# Call the function
func(dir)
else:
# Additional check that wasn't allowed to create this dir and became unmounted.
if last_mount_point_for_dir is not None:
dirs_unmounted.add(dir)
msg = "Directory {0} does not exist and became unmounted from {1} .".format(
dir, last_mount_point_for_dir)
error_messages.append(msg)
pass
# This is set to false during unit tests.
if update_cache:
get_and_cache_mount_points(refresh=True)
# Update all dirs (except the unmounted ones) with their current mount points.
for dir in valid_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(dir) and dir not in dirs_unmounted:
curr_mount_point = get_mount_point_for_dir(dir)
dir_to_mount_point[dir] = curr_mount_point
func(dir)
if error_messages and len(error_messages) > 0:
header = " ERROR ".join(["*****"] * 6)
header = "\n" + "\n".join([
header,
] * 3) + "\n"
msg = " ".join(error_messages) + \
" Please remount the dir(s) and run this command again. To ignore this failure and allow writing to the " \
"root partition, either update the contents of {0}, or delete that file.".format(history_filename)
Logger.error(header + msg + header)
dir_to_mount = DIR_TO_MOUNT_HEADER
for kv in dir_to_mount_point.iteritems():
dir_to_mount += kv[0] + "," + kv[1] + "\n"
return dir_to_mount
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
warnings = []
errors = []
if configurations is None:
return (RESULT_STATE_UNKNOWN, ['There were no configurations supplied to the script.'])
# Check required properties
if DFS_DATA_DIR not in configurations:
return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(DFS_DATA_DIR)])
dfs_data_dir = configurations[DFS_DATA_DIR]
if dfs_data_dir is None:
return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script and the value is null'.format(DFS_DATA_DIR)])
data_dir_mount_file_exists = True
# This follows symlinks and will return False for a broken link (even in the middle of the linked list)
if not os.path.exists(DATA_DIR_MOUNT_FILE):
data_dir_mount_file_exists = False
warnings.append("File not found, {0} .".format(DATA_DIR_MOUNT_FILE))
valid_data_dirs = set() # data dirs that have been normalized
data_dirs_not_exist = set() # data dirs that do not exist
data_dirs_unknown = set() # data dirs for which could not determine mount
data_dirs_on_root = set() # set of data dirs that are on root mount
data_dirs_on_mount = set() # set of data dirs that are mounted on a device
data_dirs_unmounted = [] # list of data dirs that are known to have become unmounted
for data_dir in dfs_data_dir.split(","):
if data_dir is None or data_dir.strip() == "":
continue
data_dir = data_dir.strip()
valid_data_dirs.add(data_dir)
# Sort the data dirs, which is needed for deterministic behavior when running the unit tests.
valid_data_dirs = sorted(valid_data_dirs)
for data_dir in valid_data_dirs:
# This follows symlinks and will return False for a broken link (even in the middle of the linked list)
if os.path.isdir(data_dir):
curr_mount_point = file_system.get_mount_point_for_dir(data_dir)
curr_mount_point = curr_mount_point.strip() if curr_mount_point else curr_mount_point
if curr_mount_point is not None and curr_mount_point != "":
if curr_mount_point == "/":
data_dirs_on_root.add(data_dir)
else:
data_dirs_on_mount.add(data_dir)
else:
data_dirs_unknown.add(data_dir)
else:
data_dirs_not_exist.add(data_dir)
# To keep the messages consistent for all hosts, sort the sets into lists
valid_data_dirs = sorted(valid_data_dirs)
data_dirs_not_exist = sorted(data_dirs_not_exist)
data_dirs_unknown = sorted(data_dirs_unknown)
data_dirs_on_root = sorted(data_dirs_on_root)
if data_dirs_not_exist:
errors.append("Data dir(s) not found: {0} .".format(", ".join(data_dirs_not_exist)))
if data_dirs_unknown:
errors.append("Cannot find mount point for data dir(s): {0} .".format(", ".join(data_dirs_unknown)))
if data_dir_mount_file_exists:
# Make a precise determination on which data dirs have become unmounted.
class Params:
def __init__(self, mount_file):
self.data_dir_mount_file = mount_file
params = Params(DATA_DIR_MOUNT_FILE)
# This dictionary contains the expected values of <data_dir, mount_point>
# Hence, we only need to analyze the data dirs that are currently on the root partition
# and report an error if they were expected to be on a mount.
#
# If one of the data dirs is not present in the file, it means that DataNode has not been restarted after
# the configuration was changed on the server, so we cannot make any assertions about it.
expected_data_dir_to_mount = dfs_datanode_helper.get_data_dir_to_mount_from_file(params)
for data_dir in data_dirs_on_root:
if data_dir in expected_data_dir_to_mount and expected_data_dir_to_mount[data_dir] != "/":
data_dirs_unmounted.append(data_dir)
if len(data_dirs_unmounted) > 0:
errors.append("Detected data dir(s) that became unmounted and are now writing to the root partition: {0} .".format(", ".join(data_dirs_unmounted)))
else:
# Couldn't make guarantees about the expected value of mount points, so rely on this strategy that is likely to work.
# It will report false positives (aka false alarms) if the user actually intended to have
# 1+ data dirs on a mount and 1+ data dirs on the root partition.
if len(data_dirs_on_mount) >= 1 and len(data_dirs_on_root) >= 1:
errors.append("Detected at least one data dir on a mount point, but these are writing to the root partition: {0} .".format(", ".join(data_dirs_on_root)))
# Determine the status based on warnings and errors.
if len(errors) == 0:
status = RESULT_STATE_OK
messages = []
# Check for warnings
if len(warnings) > 0:
status = RESULT_STATE_WARNING
messages += warnings
if len(valid_data_dirs) > 0:
messages.append("Data dir(s) are fine, {0} .".format(", ".join(valid_data_dirs)))
else:
messages.append("No data dirs to analyze.")
return (status, ["\n".join(messages)])
else:
# Report errors
return (RESULT_STATE_CRITICAL, ["\n".join(errors)])
def handle_mounted_dirs(func,
dirs_string,
history_filename,
update_cache=True):
"""
This function determine which dir paths can be created.
There are 2 uses cases:
1. Customers that have many dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all dirs are mounted on the root.
The goal is to avoid forcefully creating a dir when a user's drive fails. In this scenario, the
mount point for a dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the history_file parameter to parse a file that contains
a mapping from a dir, and its last known mount point.
After determining which dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(dir)
:param update_cache: Bool indicating whether to update the global cache of mount points
:return: Returns a history_filename content
"""
Directory(
os.path.dirname(history_filename),
create_parents=True,
mode=0755,
)
# Get the dirs that Ambari knows about and their last known mount point
prev_dir_to_mount_point = get_dir_to_mount_from_file(history_filename)
# Dictionary from dir to the mount point that will be written to the history file.
# If a dir becomes unmounted, we should still keep its original value.
# If a dir was previously on / and is now mounted on a drive, we should store that too.
dir_to_mount_point = prev_dir_to_mount_point.copy()
# This should typically be True after first DataNode start, but False the first time.
history_file_exists = True
if history_filename is None:
history_file_exists = False
Logger.warning("History_file.file property is null.")
else:
if not os.path.exists(history_filename):
history_file_exists = False
Logger.warning(
"History_file property has file %s and it does not exist." %
history_filename)
valid_dirs = [] # dirs that have been normalized
error_messages = [] # list of error messages to report at the end
dirs_unmounted = set() # set of dirs that have become unmounted
valid_existing_dirs = []
dirs_string = ",".join([
re.sub(r'^\[.+\]', '', dfs_dir.strip())
for dfs_dir in dirs_string.split(",")
])
for dir in dirs_string.split(","):
if dir is None or dir.strip() == "":
continue
dir = dir.strip()
valid_dirs.append(dir)
if os.path.isdir(dir):
valid_existing_dirs.append(dir)
used_mounts = set(
[get_mount_point_for_dir(dir) for dir in valid_existing_dirs])
ignore_bad_mounts = default(
'/configurations/cluster-env/ignore_bad_mounts', False)
manage_dirs_on_root = default(
'/configurations/cluster-env/manage_dirs_on_root', True)
for dir_ in valid_dirs:
last_mount_point_for_dir = prev_dir_to_mount_point.get(
dir_, None) if history_file_exists else None
curr_mount_point = get_mount_point_for_dir(dir_)
is_non_root_dir = curr_mount_point is not None and curr_mount_point != "/"
folder_exists = dir_ in valid_existing_dirs
if not folder_exists and ignore_bad_mounts:
Logger.debug("The directory {0} doesn't exist.".format(dir_))
Logger.warning(
"Not creating {0} as cluster-env/ignore_bad_mounts is enabled."
.format(dir_))
may_manage_this_dir = False
else:
may_manage_this_dir = _may_manage_folder(
dir_, last_mount_point_for_dir, is_non_root_dir,
dirs_unmounted, error_messages, manage_dirs_on_root,
curr_mount_point)
if may_manage_this_dir and dir_ not in valid_existing_dirs and curr_mount_point in used_mounts:
if default('/configurations/cluster-env/one_dir_per_partition',
False):
may_manage_this_dir = False
Logger.warning(
"Skipping creation of another directory on the following mount: "
+ curr_mount_point +
" . Please turn off cluster-env/one_dir_per_partition or handle the situation manually."
)
else:
Logger.warning(
"Trying to create another directory on the following mount: "
+ str(curr_mount_point))
if may_manage_this_dir:
Logger.info(
"Forcefully ensuring existence and permissions of the directory: {0}"
.format(dir_))
# Call the function
func(dir_)
used_mounts.add(curr_mount_point)
pass
# This is set to false during unit tests.
if update_cache:
get_and_cache_mount_points(refresh=True)
# Update all dirs (except the unmounted ones) with their current mount points.
for dir in valid_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(dir) and dir not in dirs_unmounted:
curr_mount_point = get_mount_point_for_dir(dir)
dir_to_mount_point[dir] = curr_mount_point
if error_messages and len(error_messages) > 0:
header = " WARNING ".join(["*****"] * 6)
header = "\n" + "\n".join([
header,
] * 3) + "\n"
msg = " ".join(error_messages) + \
" Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of {0}.".format(history_filename)
Logger.error(header + msg + header)
dir_to_mount = DIR_TO_MOUNT_HEADER
for kv in dir_to_mount_point.iteritems():
dir_to_mount += kv[0] + "," + kv[1] + "\n"
return dir_to_mount
def handle_dfs_data_dir(func, params):
"""
This function determine which DFS data dir paths can be created.
There are 2 uses cases:
1. Customers that have many DFS data dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all DFS data dirs are mounted on the root.
The goal is to avoid forcefully creating a DFS data dir when a user's drive fails. In this scenario, the
mount point for a DFS data dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the dfs.datanode.data.dir.mount.file parameter to parse a file that contains
a mapping from a DFS data dir, and its last known mount point.
After determining which DFS data dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(data_dir, params)
:param params: parameters to pass to function pointer
"""
prev_data_dir_to_mount_point = _get_data_dir_to_mount_from_file()
allowed_to_create_any_dir = params.data_dir_mount_file is None or not os.path.exists(params.data_dir_mount_file)
valid_data_dirs = []
for data_dir in params.dfs_data_dir.split(","):
if data_dir is None or data_dir.strip() == "":
continue
data_dir = data_dir.strip()
valid_data_dirs.append(data_dir)
if not os.path.isdir(data_dir):
create_this_dir = allowed_to_create_any_dir
# Determine if should be allowed to create the data_dir directory
if not create_this_dir:
last_mount_point_for_dir = prev_data_dir_to_mount_point[data_dir] if data_dir in prev_data_dir_to_mount_point else None
if last_mount_point_for_dir is None:
# Couldn't retrieve any information about where this dir used to be mounted, so allow creating the directory
# to be safe.
create_this_dir = True
else:
curr_mount_point = get_mount_point_for_dir(data_dir)
# This means that create_this_dir will stay false if the directory became unmounted.
if last_mount_point_for_dir == "/" or (curr_mount_point is not None and curr_mount_point != "/"):
create_this_dir = True
if create_this_dir:
Logger.info("Forcefully creating directory: %s" % str(data_dir))
# Call the function
func(data_dir, params)
else:
Logger.warning("Directory %s does not exist and became unmounted." % str(data_dir))
# Refresh the known mount points
get_and_cache_mount_points(refresh=True)
new_data_dir_to_mount_point = {}
for data_dir in valid_data_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(data_dir):
curr_mount_point = get_mount_point_for_dir(data_dir)
new_data_dir_to_mount_point[data_dir] = curr_mount_point
# Save back to the file
_write_data_dir_to_mount_in_file(new_data_dir_to_mount_point)
def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True):
"""
This function determine which dir paths can be created.
There are 2 uses cases:
1. Customers that have many dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all dirs are mounted on the root.
The goal is to avoid forcefully creating a dir when a user's drive fails. In this scenario, the
mount point for a dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the history_file parameter to parse a file that contains
a mapping from a dir, and its last known mount point.
After determining which dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(dir)
:param update_cache: Bool indicating whether to update the global cache of mount points
:return: Returns a history_filename content
"""
Directory(os.path.dirname(history_filename),
create_parents = True,
mode=0755,
)
# Get the dirs that Ambari knows about and their last known mount point
prev_dir_to_mount_point = get_dir_to_mount_from_file(history_filename)
# Dictionary from dir to the mount point that will be written to the history file.
# If a dir becomes unmounted, we should still keep its original value.
# If a dir was previously on / and is now mounted on a drive, we should store that too.
dir_to_mount_point = prev_dir_to_mount_point.copy()
# This should typically be True after first DataNode start, but False the first time.
history_file_exists = True
if history_filename is None:
history_file_exists = False
Logger.warning("History_file.file property is null.")
else:
if not os.path.exists(history_filename):
history_file_exists = False
Logger.warning("History_file property has file %s and it does not exist." % history_filename)
valid_dirs = [] # dirs that have been normalized
error_messages = [] # list of error messages to report at the end
dirs_unmounted = set() # set of dirs that have become unmounted
valid_existing_dirs = []
dirs_string = ",".join([re.sub(r'^\[.+\]', '', dfs_dir.strip()) for dfs_dir in dirs_string.split(",")])
for dir in dirs_string.split(","):
if dir is None or dir.strip() == "":
continue
dir = dir.strip()
valid_dirs.append(dir)
if os.path.isdir(dir):
valid_existing_dirs.append(dir)
used_mounts = set([get_mount_point_for_dir(dir) for dir in valid_existing_dirs])
ignore_bad_mounts = default('/configurations/cluster-env/ignore_bad_mounts', False)
manage_dirs_on_root = default('/configurations/cluster-env/manage_dirs_on_root', True)
for dir_ in valid_dirs:
last_mount_point_for_dir = prev_dir_to_mount_point.get(dir_, None) if history_file_exists else None
curr_mount_point = get_mount_point_for_dir(dir_)
is_non_root_dir = curr_mount_point is not None and curr_mount_point != "/"
folder_exists = dir_ in valid_existing_dirs
if not folder_exists and ignore_bad_mounts:
Logger.debug("The directory {0} doesn't exist.".format(dir_))
Logger.warning("Not creating {0} as cluster-env/ignore_bad_mounts is enabled.".format(dir_))
may_manage_this_dir = False
else:
may_manage_this_dir = _may_manage_folder(dir_, last_mount_point_for_dir, is_non_root_dir, dirs_unmounted, error_messages, manage_dirs_on_root, curr_mount_point)
if may_manage_this_dir and dir_ not in valid_existing_dirs and curr_mount_point in used_mounts:
if default('/configurations/cluster-env/one_dir_per_partition', False):
may_manage_this_dir = False
Logger.warning("Skipping creation of another directory on the following mount: " + curr_mount_point + " . Please turn off cluster-env/one_dir_per_partition or handle the situation manually.")
else:
Logger.warning("Trying to create another directory on the following mount: " + str(curr_mount_point))
if may_manage_this_dir:
Logger.info("Forcefully ensuring existence and permissions of the directory: {0}".format(dir_))
# Call the function
func(dir_)
used_mounts.add(curr_mount_point)
pass
# This is set to false during unit tests.
if update_cache:
get_and_cache_mount_points(refresh=True)
# Update all dirs (except the unmounted ones) with their current mount points.
for dir in valid_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(dir) and dir not in dirs_unmounted:
curr_mount_point = get_mount_point_for_dir(dir)
dir_to_mount_point[dir] = curr_mount_point
if error_messages and len(error_messages) > 0:
header = " WARNING ".join(["*****"] * 6)
header = "\n" + "\n".join([header, ] * 3) + "\n"
msg = " ".join(error_messages) + \
" Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of {0}.".format(history_filename)
Logger.error(header + msg + header)
dir_to_mount = DIR_TO_MOUNT_HEADER
for kv in dir_to_mount_point.iteritems():
dir_to_mount += kv[0] + "," + kv[1] + "\n"
return dir_to_mount
def handle_dfs_data_dir(func, params, update_cache=True):
"""
This function determine which DFS data dir paths can be created.
There are 2 uses cases:
1. Customers that have many DFS data dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all DFS data dirs are mounted on the root.
The goal is to avoid forcefully creating a DFS data dir when a user's drive fails. In this scenario, the
mount point for a DFS data dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the dfs.datanode.data.dir.mount.file parameter to parse a file that contains
a mapping from a DFS data dir, and its last known mount point.
After determining which DFS data dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(data_dir, params)
:param params: parameters to pass to function pointer
:param update_cache: Bool indicating whether to update the global cache of mount points
:return: Returns a data_dir_mount_file content
"""
# Get the data dirs that Ambari knows about and their last known mount point
prev_data_dir_to_mount_point = get_data_dir_to_mount_from_file(params)
# Dictionary from data dir to the mount point that will be written to the history file.
# If a data dir becomes unmounted, we should still keep its original value.
# If a data dir was previously on / and is now mounted on a drive, we should store that too.
data_dir_to_mount_point = prev_data_dir_to_mount_point.copy()
# This should typically be False for customers, but True the first time.
allowed_to_create_any_dir = False
if params.data_dir_mount_file is None:
allowed_to_create_any_dir = True
Logger.warning("DataNode is allowed to create any data directory since dfs.datanode.data.dir.mount.file property is null.")
else:
if not os.path.exists(params.data_dir_mount_file):
allowed_to_create_any_dir = True
Logger.warning("DataNode is allowed to create any data directory since dfs.datanode.data.dir.mount.file property has file %s and it does not exist." % params.data_dir_mount_file)
valid_data_dirs = [] # data dirs that have been normalized
error_messages = [] # list of error messages to report at the end
data_dirs_unmounted = set() # set of data dirs that have become unmounted
for data_dir in params.dfs_data_dir.split(","):
if data_dir is None or data_dir.strip() == "":
continue
data_dir = data_dir.strip()
valid_data_dirs.append(data_dir)
if not os.path.isdir(data_dir):
may_create_this_dir = allowed_to_create_any_dir
last_mount_point_for_dir = None
# Determine if should be allowed to create the data_dir directory.
# Either first time, became unmounted, or was just mounted on a drive
if not may_create_this_dir:
last_mount_point_for_dir = prev_data_dir_to_mount_point[data_dir] if data_dir in prev_data_dir_to_mount_point else None
if last_mount_point_for_dir is None:
# Couldn't retrieve any information about where this dir used to be mounted, so allow creating the directory to be safe.
may_create_this_dir = True
else:
curr_mount_point = get_mount_point_for_dir(data_dir)
# This means that create_this_dir will stay false if the directory became unmounted.
# In other words, allow creating if it was already on /, or it's currently not on /
if last_mount_point_for_dir == "/" or (curr_mount_point is not None and curr_mount_point != "/"):
may_create_this_dir = True
if may_create_this_dir:
Logger.info("Forcefully creating directory: {0}".format(data_dir))
# Call the function
func(data_dir, params)
else:
# Additional check that wasn't allowed to create this dir and became unmounted.
if last_mount_point_for_dir is not None:
data_dirs_unmounted.add(data_dir)
msg = "Directory {0} does not exist and became unmounted from {1} .".format(data_dir, last_mount_point_for_dir)
error_messages.append(msg)
pass
# This is set to false during unit tests.
if update_cache:
get_and_cache_mount_points(refresh=True)
# Update all data dirs (except the unmounted ones) with their current mount points.
for data_dir in valid_data_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(data_dir) and data_dir not in data_dirs_unmounted:
curr_mount_point = get_mount_point_for_dir(data_dir)
data_dir_to_mount_point[data_dir] = curr_mount_point
if error_messages and len(error_messages) > 0:
header = " ERROR ".join(["*****"] * 6)
header = "\n" + "\n".join([header, ] * 3) + "\n"
msg = " ".join(error_messages) + \
" Please remount the data dir(s) and run this command again. To ignore this failure and allow writing to the " \
"root partition, either update the contents of {0}, or delete that file.".format(params.data_dir_mount_file)
Logger.error(header + msg + header)
data_dir_to_mount = DATA_DIR_TO_MOUNT_HEADER
for kv in data_dir_to_mount_point.iteritems():
data_dir_to_mount += kv[0] + "," + kv[1] + "\n"
return data_dir_to_mount
def handle_dfs_data_dir(func, params):
"""
This function determine which DFS data dir paths can be created.
There are 2 uses cases:
1. Customers that have many DFS data dirs, each one on a separate mount point that corresponds to a different drive.
2. Developers that are using a sandbox VM and all DFS data dirs are mounted on the root.
The goal is to avoid forcefully creating a DFS data dir when a user's drive fails. In this scenario, the
mount point for a DFS data dir changes from something like /hadoop/hdfs/data/data1 to /
If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon
fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure
based on its tolerance of missing directories.
This function relies on the dfs.datanode.data.dir.mount.file parameter to parse a file that contains
a mapping from a DFS data dir, and its last known mount point.
After determining which DFS data dirs can be created if they don't exist, it recalculates the mount points and
writes to the file again.
:param func: Function that will be called if a directory will be created. This function
will be called as func(data_dir, params)
:param params: parameters to pass to function pointer
"""
prev_data_dir_to_mount_point = _get_data_dir_to_mount_from_file()
allowed_to_create_any_dir = params.data_dir_mount_file is None or not os.path.exists(params.data_dir_mount_file)
valid_data_dirs = []
for data_dir in params.dfs_data_dir.split(","):
if data_dir is None or data_dir.strip() == "":
continue
data_dir = data_dir.strip().lower()
valid_data_dirs.append(data_dir)
if not os.path.isdir(data_dir):
create_this_dir = allowed_to_create_any_dir
# Determine if should be allowed to create the data_dir directory
if not create_this_dir:
last_mount_point_for_dir = prev_data_dir_to_mount_point[data_dir] if data_dir in prev_data_dir_to_mount_point else None
if last_mount_point_for_dir is None:
# Couldn't retrieve any information about where this dir used to be mounted, so allow creating the directory
# to be safe.
create_this_dir = True
else:
curr_mount_point = get_mount_point_for_dir(data_dir)
# This means that create_this_dir will stay false if the directory became unmounted.
if last_mount_point_for_dir == "/" or (curr_mount_point is not None and curr_mount_point != "/"):
create_this_dir = True
if create_this_dir:
Logger.info("Forcefully creating directory: %s" % str(data_dir))
# Call the function
func(data_dir, params)
else:
Logger.warning("Directory %s does not exist and became unmounted." % str(data_dir))
# Refresh the known mount points
get_and_cache_mount_points(refresh=True)
new_data_dir_to_mount_point = {}
for data_dir in valid_data_dirs:
# At this point, the directory may or may not exist
if os.path.isdir(data_dir):
curr_mount_point = get_mount_point_for_dir(data_dir)
new_data_dir_to_mount_point[data_dir] = curr_mount_point
# Save back to the file
_write_data_dir_to_mount_in_file(new_data_dir_to_mount_point)
