def _sync_data_block_meta(self, index): if self._dumped_index < 0 or index > self._dumped_index: return None if index not in self._data_block_meta_cache: self._evict_data_block_cache_if_full() fpath = self._get_data_block_meta_path(index) meta = load_data_block_meta(fpath) if meta is None: logging.fatal("data block index as %d has dumped "\ "but vanish", index) os._exit(-1) # pylint: disable=protected-access self._data_block_meta_cache[index] = meta return self._data_block_meta_cache[index]
def LoadDataBlockReqByIndex(self, partition_id, data_block_index): partition_num = self._data_source.data_source_meta.partition_num if partition_id < 0 or partition_id >= partition_num: raise IndexError("partition {} out range".format(partition_id)) dirpath = self._partition_data_block_dir(partition_id) meta_fname = encode_data_block_meta_fname(self._data_source_name(), partition_id, data_block_index) meta_fpath = os.path.join(dirpath, meta_fname) meta = load_data_block_meta(meta_fpath) manifest = self._sync_raw_data_manifest(partition_id) if meta is not None and \ not self._filter_by_visible(meta.data_block_index, manifest): fname = encode_data_block_fname(self._data_source_name(), meta) return DataBlockRep(self._data_source_name(), fname, partition_id, dirpath) return None
def data_block_meta(self): if self._data_block_meta is None: self._data_block_meta = \ load_data_block_meta(self._data_block_meta_fpath) return self._data_block_meta