def _parse_pack(self, multi_pack_str: str) -> Iterator[MultiPack]:
# pylint: disable=protected-access
m_pack: MultiPack = deserialize(multi_pack_str)
for pid in m_pack._pack_ref:
pack: DataPack = deserialize(self._get_pack_content(pid))
m_pack._packs.append(pack)
yield m_pack
def _parse_pack(self, multi_pack_path: str) -> Iterator[MultiPack]:
# pylint: disable=protected-access
with open(os.path.join(self.configs.data_path,
multi_pack_path)) as m_data:
m_pack: MultiPack = deserialize(m_data.read())
for pid in m_pack._pack_ref:
sub_pack_path = self.__pack_index[pid]
with open(os.path.join(self.configs.data_path,
sub_pack_path)) as pack_data:
pack: DataPack = deserialize(pack_data.read())
m_pack._packs.append(pack)
yield m_pack
def _parse_pack(
self, collection: Tuple[str, Dict[str, List[state_type]]]
) -> Iterator[DataPack]:
resource_name, info_box_data = collection
if resource_name in self.redirects:
resource_name = self.redirects[resource_name]
if resource_name in self.pack_index:
print_progress(f'Add infobox to resource: [{resource_name}]')
pack_path = os.path.join(
self.pack_dir,
self.pack_index[resource_name]
)
if os.path.exists(pack_path):
with open(pack_path) as pack_file:
pack = data_utils.deserialize(
self._pack_manager, pack_file.read())
add_info_boxes(pack, info_box_data['literals'])
add_info_boxes(pack, info_box_data['objects'])
add_property(pack, info_box_data['properties'])
yield pack
else:
print_notice(f"Resource {resource_name} is not in the raw packs.")
self.logger.warning("Resource %s is not in the raw packs.",
resource_name)
def _parse_pack(self, data_source: str) -> Iterator[DataPack]:
if data_source is None:
raise ProcessExecutionException(
"Data source is None, cannot deserialize.")
# pack: DataPack = DataPack.deserialize(data_source)
pack: DataPack = deserialize(self._pack_manager, data_source)
if pack is None:
raise ProcessExecutionException(
f"Cannot recover pack from the following data source: \n"
f"{data_source}")
yield pack
def _parse_pack(self, multi_pack_path: str) -> Iterator[MultiPack]:
# pylint: disable=protected-access
with open(os.path.join(self.configs.data_path,
multi_pack_path)) as m_data:
# m_pack: MultiPack = MultiPack.deserialize(m_data.read())
m_pack: MultiPack = deserialize(self._pack_manager, m_data.read())
for pid in m_pack._pack_ref:
sub_pack_path = self.__pack_index[pid]
if self._pack_manager.get_remapped_id(pid) >= 0:
# This pid is already been read.
continue
with open(os.path.join(self.configs.data_path,
sub_pack_path)) as pack_data:
# pack: DataPack = DataPack.deserialize(pack_data.read())
pack: DataPack = deserialize(self._pack_manager,
pack_data.read())
# Add a reference count to this pack, because the multipack
# needs it.
self._pack_manager.reference_pack(pack)
m_pack.realign_packs()
yield m_pack
def read_from_cache(
self, cache_filename: Union[Path, str]) -> Iterator[PackType]:
r"""Reads one or more Packs from ``cache_filename``, and yields Pack(s)
from the cache file.
Args:
cache_filename: Path to the cache file.
Returns: List of cached data packs.
"""
logger.info("reading from cache file %s", cache_filename)
with open(cache_filename, "r") as cache_file:
for line in cache_file:
pack = data_utils.deserialize(line.strip())
if not isinstance(pack, self.pack_type):
raise TypeError(
f"Pack deserialized from {cache_filename} "
f"is {type(pack)}, but expect {self.pack_type}")
yield pack
