def _sync_data_block_meta(self, index):
if self._dumped_index < 0 or index > self._dumped_index:
return None
if index not in self._data_block_meta_cache:
self._evict_data_block_cache_if_full()
fpath = self._get_data_block_meta_path(index)
meta = load_data_block_meta(fpath)
if meta is None:
logging.fatal("data block index as %d has dumped "\
"but vanish", index)
os._exit(-1) # pylint: disable=protected-access
self._data_block_meta_cache[index] = meta
return self._data_block_meta_cache[index]
def LoadDataBlockReqByIndex(self, partition_id, data_block_index):
partition_num = self._data_source.data_source_meta.partition_num
if partition_id < 0 or partition_id >= partition_num:
raise IndexError("partition {} out range".format(partition_id))
dirpath = self._partition_data_block_dir(partition_id)
meta_fname = encode_data_block_meta_fname(self._data_source_name(),
partition_id,
data_block_index)
meta_fpath = os.path.join(dirpath, meta_fname)
meta = load_data_block_meta(meta_fpath)
manifest = self._sync_raw_data_manifest(partition_id)
if meta is not None and \
not self._filter_by_visible(meta.data_block_index, manifest):
fname = encode_data_block_fname(self._data_source_name(), meta)
return DataBlockRep(self._data_source_name(), fname, partition_id,
dirpath)
return None
def data_block_meta(self):
if self._data_block_meta is None:
self._data_block_meta = \
load_data_block_meta(self._data_block_meta_fpath)
return self._data_block_meta
