def deserialize(pack_manager: PackManager, string: str):
r"""Deserialize a pack from a string.
"""
pack = jsonpickle.decode(string)
# Need to assign the pack manager to the pack to control it after reading
# the raw data.
# pylint: disable=protected-access
pack._pack_manager = pack_manager
pack_manager.set_remapped_pack_id(pack)
return pack
def __init__(self, resource: Optional[Resources] = None):
self._reader: BaseReader
self._reader_config: Optional[Config]
self._components: List[PipelineComponent] = []
self._selectors: List[Selector] = []
self._processors_index: Dict = {'': -1}
self._configs: List[Optional[Config]] = []
# Will intialize at `initialize` because the processors length is
# unknown.
self.proc_mgr: _ProcessManager
# This manager controls global pack access information
self._pack_manager: PackManager = PackManager()
self._pack_manager.reset()
self.evaluator_indices: List[int] = []
# needed for evaluator
self._predict_to_gold: Dict[int, PackType] = {}
if resource is None:
self.resource = Resources()
else:
self.resource = resource
self.initialized: bool = False
def test_wrong_attribute(self):
manager = PackManager()
input_pack = MultiPack(manager)
mp_entry = ExampleMPEntry(input_pack)
p1 = input_pack.add_pack('pack1')
e1: DifferentEntry = p1.add_entry(DifferentEntry(p1))
with self.assertRaises(TypeError):
mp_entry.refer_entry = e1
mp_entry.regret_creation()
def __setstate__(self, state):
"""
Re-obtain the pack manager during deserialization.
Args:
state:
Returns:
"""
self.__dict__.update(state)
self._pack_manager: PackManager = PackManager()
def setUp(self) -> None:
pm = PackManager()
self.multi_pack = MultiPack(pm)
data_pack1 = self.multi_pack.add_pack(ref_name="pack1")
data_pack2 = self.multi_pack.add_pack(ref_name="pack2")
data_pack3 = self.multi_pack.add_pack(ref_name="pack_three")
data_pack1.pack_name = "1"
data_pack2.pack_name = "2"
data_pack3.pack_name = "Three"
def setUp(self) -> None:
# Note: input source is created automatically by the system, but we
# can also set it manually at test cases.
pm = PackManager()
self.multi_pack = MultiPack(pm)
self.data_pack1 = self.multi_pack.add_pack(ref_name="left pack")
self.data_pack2 = self.multi_pack.add_pack(ref_name="right pack")
self.data_pack1.pack_name = "some pack"
self.data_pack1.set_text("This pack contains some sample data.")
self.data_pack2.pack_name = "another pack"
self.data_pack2.set_text("This pack contains some other sample data.")
def __init__(self, doc_id: Optional[str] = None):
super().__init__()
self.links: List[LinkType] = []
self.groups: List[GroupType] = []
self.meta: BaseMeta = BaseMeta(doc_id)
self.index: BaseIndex = BaseIndex()
# Obtain the global pack manager.
self._pack_manager: PackManager = PackManager()
self.__control_component: Optional[str] = None
self._pending_entries: Dict[int, Tuple[Entry, str]] = {}
def test_reader_original_span_test(self, value):
span_ops, output = ([(Span(11, 19), 'New'), (Span(19, 20), ' Shiny '),
(Span(25, 25), ' Ends')],
'<title>The New Shiny Title Ends </title>')
input_span, expected_span, mode = value
reader = PlainTextReader()
PackManager().set_input_source(reader.component_name)
reader.text_replace_operation = lambda _: span_ops
pack = list(reader.parse_pack(self.file_path))[0]
self.assertEqual(pack.text, output)
output_span = pack.get_original_span(input_span, mode)
self.assertEqual(
output_span, expected_span,
f"Expected: ({expected_span.begin, expected_span.end}"
f"), Found: ({output_span.begin, output_span.end})"
f" when Input: ({input_span.begin, input_span.end})"
f" and Mode: {mode}")
def test_reader_no_replace_test(self):
# Read with no replacements
reader = RACEMultiChoiceQAReader()
PackManager().set_input_source(reader.name)
data_packs: Iterator[DataPack] = reader.iter(self.dataset_path)
file_paths: Iterator[str] = reader._collect(self.dataset_path)
count_packs = 0
for pack, file_path in zip(data_packs, file_paths):
count_packs += 1
expected_text: str = ""
with open(file_path, "r", encoding="utf8",
errors='ignore') as file:
expected = json.load(file)
articles = list(pack.get(RaceDocument))
self.assertEqual(len(articles), 1)
expected_article = expected['article']
self.assertEqual(articles[0].text, expected_article)
expected_text += expected_article
for qid, question in enumerate(pack.get(Question)):
expected_question = expected['questions'][qid]
self.assertEqual(question.text, expected_question)
expected_answers = expected['answers'][qid]
if not isinstance(expected_answers, list):
expected_answers = [expected_answers]
expected_answers = [
reader._convert_to_int(ans) for ans in expected_answers
]
self.assertEqual(question.answers, expected_answers)
expected_text += '\n' + expected_question
for oid, option in enumerate(question.options):
expected_option = expected['options'][qid][oid]
self.assertEqual(option.text, expected_option)
expected_text += '\n' + expected_option
self.assertEqual(pack.text, expected_text)
self.assertEqual(count_packs, 2)
def test_reader_no_replace_test(self):
# Read with no replacements
reader = PlainTextReader()
PackManager().set_input_source(reader.component_name)
pack = list(reader.parse_pack(self.file_path))[0]
self.assertEqual(pack.text, self.orig_text)
def __init__(self):
self._process_manager: _ProcessManager
self._pack_manager: PackManager = PackManager()
def setUp(self):
manager = PackManager()
self.pack: DataPack = DataPack(manager)
self.pack.set_text("Some text to test annotations on.")
def __init__(self, doc_id: Optional[str] = None):
self.doc_id: Optional[str] = doc_id
self._pack_id: int = -1
# Obtain the global pack manager.
self._pack_manager: PackManager = PackManager()
def __setstate__(self, state):
super().__setstate__(state)
self.__dict__['_pack_manager'] = PackManager()
self.__dict__['_pending_entries'] = {}
self.__control_component: Optional[str] = None