def make_new_partition(self, timestamp): new_dir_list = FileUtil.parse_timestamp_to_dir( timestamp=timestamp).split('/') new_dir = '/'.join(new_dir_list[:self.PARTITIONER_TYPE_TO_HEIGHT_MAP[ self.PARTITIONER_TYPE]]) child_node = OrderedNodeBase(node_name=FileUtil.join_paths_to_dir( root_dir=self._file_tree.get_root_name(), base_name=new_dir)) if FileUtil.does_dir_exist(dir_name=child_node.get_node_name()): self.sys_log('Node [' + child_node.get_node_name() + "] exist. Don't make new partition.") return None else: self.sys_log('Node [' + child_node.get_node_name() + "] doesn't exist. Make new partition.") self._logger.info('Node [' + child_node.get_node_name() + "] doesn't exist. Make new partition.") FileUtil.create_dir_if_not_exist( dir_name=child_node.get_node_name()) self.initialize_from_dir(dir_name=self._file_tree.get_root_name()) return child_node.get_node_name()
def initialize_from_dir(self, dir_name, force=False): def _recursive_initialize_from_dir(node, max_recursion): self.sys_log("Starting recursion of " + str(max_recursion) + '.') if max_recursion == 0: self.sys_log("Exhausted all recursions for dir [" + dir_name + '].') self._logger.info("Exhausted all recursions for dir [" + dir_name + '].') return node_name = node.get_node_name() for child_node_name in sorted( FileUtil.list_dirs_in_dir(dir_name=node_name), reverse=from_scratch): if from_scratch and self._file_tree.get_num_nodes( ) >= self._max_capacity > 0: self.sys_log("Reach the max number of node: " + str(self._max_capacity) + '.') return newly_added_string = child_node_name.replace(node_name, '').replace( '/', '') if not newly_added_string.isdigit(): continue if not from_scratch and self._cmp_dir_by_timestamp( dir_name_1=child_node_name, dir_name_2=self._get_latest_dir_internal()): continue child_node = self._file_tree.find_node( node_name=child_node_name) if not child_node: child_node = OrderedNodeBase(node_name=child_node_name) # The nodes are ordered from large to small. So if the tree is built scratch, since the directory # is listed from large to small, SortOrder.ORDER is used. If it is incremental build, since the # directory is listed from small to large, SortOrder.REVERSE is used. order = SortOrder.ORDER if from_scratch else SortOrder.REVERSE self._file_tree.add_node(parent_node=node, child_node=child_node, order=order) self.sys_log("Adding new node [" + child_node_name + node.get_node_name() + '].') self._logger.info("Adding new node [" + child_node_name + "] to parent node [" + node.get_node_name() + '].') if not from_scratch: self._file_tree.trim_tree( max_capacity=self._max_capacity) _recursive_initialize_from_dir(node=child_node, max_recursion=max_recursion - 1) from_scratch = False dir_name = FileUtil.normalize_dir_name(dir_name=dir_name) FileUtil.create_dir_if_not_exist(dir_name=dir_name) if not self._file_tree or self.is_updated() or force: root_node = OrderedNodeBase(node_name=FileUtil.normalize_dir_name( dir_name=dir_name)) self._file_tree = TreeBase(root=root_node, max_dict_size=self._max_capacity) from_scratch = True _recursive_initialize_from_dir( node=self._file_tree.get_root_node(), max_recursion=self.PARTITIONER_TYPE_TO_HEIGHT_MAP[ self.PARTITIONER_TYPE])