def setUp(self) -> None: self.data_pack1 = DataPack(doc_id="1") self.data_pack2 = DataPack(doc_id="2") self.data_pack3 = DataPack(doc_id="Three") self.multi_pack = MultiPack() self.multi_pack.add_pack(self.data_pack1, pack_name="pack1") self.multi_pack.add_pack(self.data_pack2, pack_name="pack2") self.multi_pack.add_pack(self.data_pack3, pack_name="pack_three")
def _parse_pack( self, doc_data: Tuple[Dict[str, str], Dict[str, List[state_type]]] ) -> Iterator[DataPack]: str_data, node_data = doc_data pack = DataPack() doc_name: str = str_data['doc_name'] if doc_name in self.redirects: doc_name = self.redirects[doc_name] full_text: str = str_data['text'] pack.set_text(full_text) page = WikiPage(pack, 0, len(full_text)) pack.add_entry(page) page.set_page_id(str_data['oldid']) page.set_page_name(doc_name) if len(node_data['struct']) > 0: add_struct(pack, node_data['struct']) else: logging.warning('Structure info for %s not found.', doc_name) if len(node_data['links']) > 0: add_anchor_links(pack, node_data['links'], self.redirects) else: logging.warning('Links for [%s] not found.', doc_name) pack.meta.doc_id = doc_name yield pack
def _parse_pack(self, nli_instance) -> Iterator[PackType]: pair_id, source, target, label = nli_instance pack = DataPack(pair_id) text = source + "\n" + target + "\n" pack.set_text(text) premise = Premise(pack, 0, len(source)) hypo = Hypothesis(pack, len(source) + 1, len(text) - 1) pair = NLIPair(pack) pair.set_parent(premise) pair.set_child(hypo) pair.entailment = { "entailment": 0, "neutral": 0, "contradiction": 0, } if label == 2: pair.entailment["contradiction"] = 1 elif label == 0: pair.entailment["entailment"] = 1 elif label == 1: pair.entailment["neutral"] = 1 else: raise ValueError("Unknown label value.") yield pack
def _process(self, input_pack: MultiPack): r"""Searches ElasticSearch indexer to fetch documents for a query. This query should be contained in the input multipack with name `self.config.query_pack_name`. This method adds new packs to `input_pack` containing the retrieved results. Each result is added as a `ft.onto.base_ontology.Document`. Args: input_pack: A multipack containing query as a pack. """ query_pack = input_pack.get_pack(self.config.query_pack_name) # ElasticSearchQueryCreator adds a Query entry to query pack. We now # fetch it as the first element. first_query = list(query_pack.get_entries(Query))[0] results = self.index.search(first_query.value) hits = results["hits"]["hits"] packs = {} for idx, hit in enumerate(hits): document = hit["_source"] first_query.update_results({document["doc_id"]: hit["_score"]}) pack = DataPack(doc_id=document["doc_id"]) content = document[self.config.field] document = Document(pack=pack, begin=0, end=len(content)) pack.add_entry(document) pack.set_text(content) packs[f"{self.config.response_pack_name_prefix}_{idx}"] = pack input_pack.update_pack(packs)
def _process(self, input_pack: MultiPack): query = input_pack.get_pack(self.in_pack_name).text params = '?' + urlencode( { 'api-version': '3.0', 'from': self.src_language, 'to': [self.target_language] }, doseq=True) microsoft_constructed_url = self.microsoft_translate_url + params response = requests.post(microsoft_constructed_url, headers=self.microsoft_headers, json=[{ "text": query }]) if response.status_code != 200: raise RuntimeError(response.json()['error']['message']) text = response.json()[0]["translations"][0]["text"] pack = DataPack() document = Document(pack, 0, len(text)) utterance = Utterance(pack, 0, len(text)) pack.add_entry(document) pack.add_entry(utterance) pack.set_text(text=text) input_pack.update_pack({self.out_pack_name: pack})
def _process(self, input_pack: MultiPack): query_pack = input_pack.get_pack(self.config.query_pack_name) first_query = list(query_pack.get_entries(Query))[0] results = self.index.search(first_query.value, self.k) documents = [r[1] for result in results for r in result] packs = {} for i, doc in enumerate(documents): pack = DataPack() document = Document(pack=pack, begin=0, end=len(doc)) pack.add_entry(document) pack.set_text(doc) packs[self.config.response_pack_name[i]] = pack input_pack.update_pack(packs)
def _process(self, input_pack: MultiPack): query_pack = input_pack.get_pack("pack") first_query = list(query_pack.get_entries(Query))[0] results = self.index.search(first_query.value, self.k) documents = [r[1] for result in results for r in result] packs = {} counter = 0 for doc in documents: pack = DataPack() document = Document(pack=pack, begin=0, end=len(doc)) pack.add_entry(document) pack.set_text(doc) packs[f"doc_{counter}"] = pack counter += 1 input_pack.update_pack(packs)
def _parse_pack(self, doc_info: DocInfoType) -> Iterator[DataPack]: # pylint: disable = no-self-use """ Takes the `doc_info` returned by the `_collect` method and returns a `data_pack` that either contains entry of the type `Query`, or contains an entry of the type Document. :param doc_info: document info to be populatd in the data_pack :return: query or document data_pack """ data_pack: DataPack = DataPack() is_query, doc_id, doc_content, rel_docs = doc_info data_pack.meta.doc_id = doc_id annotations: List[Union[Passage, Document, Query]] = [] if not is_query: # add passages for passage in doc_content: annotations.append(Passage(data_pack, 0, len(passage))) doc_text = os.linesep.join(doc_content) annotations.append(Document(data_pack, 0, len(doc_text))) data_pack.set_text(doc_text) else: query = Query(data_pack) query.query = doc_content[0] if rel_docs is not None: query.doc_ids = {"relevant_docs": rel_docs} annotations = [query] # add annotations to data_pack for annotation in annotations: data_pack.add_or_get_entry(annotation) yield data_pack
def _parse_pack(self, input_paths) -> Iterator[PackType]: src_path, ere_path = input_paths with open(ere_path) as f: tree = ET.parse(f) root = tree.getroot() pack = DataPack() with open(src_path) as src_text_file: src_text = replace_unicode(src_text_file.read()) self.set_text(pack, src_text) assert len(src_text) == len(pack.text) pack.pack_name = root.get("doc_id") args = [] ems = {} for c1 in root: if c1.tag == "hoppers": for c2 in c1.iter("hopper"): hopper = Hopper(pack) hopper.id = c2.get("id") for em_node in c2.iter("event_mention"): trigger = em_node.find("trigger") begin = int(trigger.get("offset")) end = begin + int(trigger.get("length")) evm = EventMention(pack, begin, end) evm.event_type = em_node.get( "type") + "_" + em_node.get("subtype") evm.realis = em_node.get("realis") evm.audience = em_node.get("audience") evm.formality = em_node.get("formality") evm.medium = em_node.get("medium") evm.schedule = em_node.get("schedule") evm.id = em_node.get("id") evm_arg = em_node.find("em_arg") if evm_arg is not None: arg_mention_id = evm_arg.get( "entity_mention_id") if arg_mention_id is None: arg_mention_id = evm_arg.get("filler_id") args.append((evm, ( arg_mention_id, evm_arg.get("role"), evm_arg.get("realis"), evm_arg.get("id"), ))) hopper.add_member(evm) elif c1.tag == "entities": for c2 in c1.iter("entity"): for em_node in c2.iter("entity_mention"): begin = int(em_node.get("offset")) end = begin + int(em_node.get("length")) em = EntityMention(pack, begin, end) em.ner_type = c1.get("type") em.id = c1.get("id") em.is_filler = False ems[em_node.get("id")] = em elif c1.tag == "fillers": for filler in c1.iter("filler"): begin = int(filler.get("offset")) end = begin + int(filler.get("length")) em = EntityMention(pack, begin, end) em.ner_type = c1.get("type") em.id = c1.get("id") em.is_filler = True ems[filler.get("id")] = em for evm, (em_id, role, realis, arg_id) in args: em = ems[em_id] argument = EventArgument(pack, evm, em) argument.role = role argument.realis = realis argument.id = arg_id yield pack
def _parse_pack(self, data_source: str) -> Iterator[DataPack]: pack = DataPack() self.set_text(pack, data_source) Utterance(pack, 0, len(data_source)) yield pack