def _parse_pack(self, multi_pack_path: str) -> Iterator[MultiPack]:
# pylint: disable=protected-access
with open(os.path.join(self.configs.data_path,
multi_pack_path)) as m_data:
m_pack: MultiPack = MultiPack.deserialize(m_data.read())
for pid in m_pack._pack_ref:
sub_pack_path = self.__pack_index[pid]
if self._pack_manager.get_remapped_id(pid) >= 0:
# This pid is already been read.
continue
with open(os.path.join(self.configs.data_path,
sub_pack_path)) as pack_data:
pack: DataPack = DataPack.deserialize(pack_data.read())
# Add a reference count to this pack, because the multipack
# needs it.
self._pack_manager.reference_pack(pack)
m_pack.realign_packs()
yield m_pack
def _parse_pack(self, multi_pack_str: str) -> Iterator[MultiPack]:
m_pack: MultiPack = MultiPack.deserialize(multi_pack_str)
for pid in m_pack.pack_ids():
p_content = self._get_pack_content(pid)
if p_content is None:
logging.warning(
"Cannot locate the data pack with pid %d "
"for multi pack %d", pid, m_pack.pack_id)
break
pack: DataPack
if isinstance(p_content, str):
pack = DataPack.deserialize(p_content)
else:
pack = p_content
# Only in deserialization we can do this.
m_pack.packs.append(pack)
else:
# No multi pack will be yield if there are packs not located.
yield m_pack
def _parse_multi_pack(self, multi_pack_source: Any) -> MultiPack:
return MultiPack.deserialize(
multi_pack_source,
self.configs.serialize_method,
self.configs.zip_pack,
)