def make_new_partition(self, timestamp):
new_dir_list = FileUtil.parse_timestamp_to_dir(
timestamp=timestamp).split('/')
new_dir = '/'.join(new_dir_list[:self.PARTITIONER_TYPE_TO_HEIGHT_MAP[
self.PARTITIONER_TYPE]])
child_node = OrderedNodeBase(node_name=FileUtil.join_paths_to_dir(
root_dir=self._file_tree.get_root_name(), base_name=new_dir))
if FileUtil.does_dir_exist(dir_name=child_node.get_node_name()):
self.sys_log('Node [' + child_node.get_node_name() +
"] exist. Don't make new partition.")
return None
else:
self.sys_log('Node [' + child_node.get_node_name() +
"] doesn't exist. Make new partition.")
self._logger.info('Node [' + child_node.get_node_name() +
"] doesn't exist. Make new partition.")
FileUtil.create_dir_if_not_exist(
dir_name=child_node.get_node_name())
self.initialize_from_dir(dir_name=self._file_tree.get_root_name())
return child_node.get_node_name()
def initialize_from_dir(self, dir_name, force=False):
def _recursive_initialize_from_dir(node, max_recursion):
self.sys_log("Starting recursion of " + str(max_recursion) + '.')
if max_recursion == 0:
self.sys_log("Exhausted all recursions for dir [" + dir_name +
'].')
self._logger.info("Exhausted all recursions for dir [" +
dir_name + '].')
return
node_name = node.get_node_name()
for child_node_name in sorted(
FileUtil.list_dirs_in_dir(dir_name=node_name),
reverse=from_scratch):
if from_scratch and self._file_tree.get_num_nodes(
) >= self._max_capacity > 0:
self.sys_log("Reach the max number of node: " +
str(self._max_capacity) + '.')
return
newly_added_string = child_node_name.replace(node_name,
'').replace(
'/', '')
if not newly_added_string.isdigit():
continue
if not from_scratch and self._cmp_dir_by_timestamp(
dir_name_1=child_node_name,
dir_name_2=self._get_latest_dir_internal()):
continue
child_node = self._file_tree.find_node(
node_name=child_node_name)
if not child_node:
child_node = OrderedNodeBase(node_name=child_node_name)
# The nodes are ordered from large to small. So if the tree is built scratch, since the directory
# is listed from large to small, SortOrder.ORDER is used. If it is incremental build, since the
# directory is listed from small to large, SortOrder.REVERSE is used.
order = SortOrder.ORDER if from_scratch else SortOrder.REVERSE
self._file_tree.add_node(parent_node=node,
child_node=child_node,
order=order)
self.sys_log("Adding new node [" + child_node_name +
node.get_node_name() + '].')
self._logger.info("Adding new node [" + child_node_name +
"] to parent node [" +
node.get_node_name() + '].')
if not from_scratch:
self._file_tree.trim_tree(
max_capacity=self._max_capacity)
_recursive_initialize_from_dir(node=child_node,
max_recursion=max_recursion - 1)
from_scratch = False
dir_name = FileUtil.normalize_dir_name(dir_name=dir_name)
FileUtil.create_dir_if_not_exist(dir_name=dir_name)
if not self._file_tree or self.is_updated() or force:
root_node = OrderedNodeBase(node_name=FileUtil.normalize_dir_name(
dir_name=dir_name))
self._file_tree = TreeBase(root=root_node,
max_dict_size=self._max_capacity)
from_scratch = True
_recursive_initialize_from_dir(
node=self._file_tree.get_root_node(),
max_recursion=self.PARTITIONER_TYPE_TO_HEIGHT_MAP[
self.PARTITIONER_TYPE])
