def _parse_pack(self, multi_pack_path: str) -> Iterator[MultiPack]: # pylint: disable=protected-access with open(os.path.join(self.configs.data_path, multi_pack_path)) as m_data: m_pack: MultiPack = MultiPack.deserialize(m_data.read()) for pid in m_pack._pack_ref: sub_pack_path = self.__pack_index[pid] if self._pack_manager.get_remapped_id(pid) >= 0: # This pid is already been read. continue with open(os.path.join(self.configs.data_path, sub_pack_path)) as pack_data: pack: DataPack = DataPack.deserialize(pack_data.read()) # Add a reference count to this pack, because the multipack # needs it. self._pack_manager.reference_pack(pack) m_pack.realign_packs() yield m_pack
def _parse_pack(self, multi_pack_str: str) -> Iterator[MultiPack]: m_pack: MultiPack = MultiPack.deserialize(multi_pack_str) for pid in m_pack.pack_ids(): p_content = self._get_pack_content(pid) if p_content is None: logging.warning( "Cannot locate the data pack with pid %d " "for multi pack %d", pid, m_pack.pack_id) break pack: DataPack if isinstance(p_content, str): pack = DataPack.deserialize(p_content) else: pack = p_content # Only in deserialization we can do this. m_pack.packs.append(pack) else: # No multi pack will be yield if there are packs not located. yield m_pack
def _parse_multi_pack(self, multi_pack_source: Any) -> MultiPack: return MultiPack.deserialize( multi_pack_source, self.configs.serialize_method, self.configs.zip_pack, )