def setup(config: Config) -> Pipeline: resource = Resources() query_pipeline = Pipeline[MultiPack](resource=resource) query_pipeline.set_reader( reader=MultiPackTerminalReader(), config=config.reader) query_pipeline.add( component=MicrosoftBingTranslator(), config=config.translator) query_pipeline.add( component=BertBasedQueryCreator(), config=config.query_creator) query_pipeline.add( component=SearchProcessor(), config=config.searcher) top_response_pack_name = config.indexer.response_pack_name + '_0' query_pipeline.add( component=NLTKSentenceSegmenter(), selector=NameMatchSelector(select_name=top_response_pack_name)) query_pipeline.add( component=NLTKWordTokenizer(), selector=NameMatchSelector(select_name=top_response_pack_name)) query_pipeline.add( component=NLTKPOSTagger(), selector=NameMatchSelector(select_name=top_response_pack_name)) query_pipeline.add( component=SRLPredictor(), config=config.SRL, selector=NameMatchSelector(select_name=top_response_pack_name)) query_pipeline.add( component=MicrosoftBingTranslator(), config=config.back_translator) query_pipeline.initialize() return query_pipeline
def setUp(self): self.kwargs = { '1': 'one', 'dummy': DummyObject(1, 2) } self.resources = Resources(**self.kwargs) self.output_dir = tempfile.mkdtemp()
def __init__(self, resource: Optional[Resources] = None): self._reader: BaseReader self._reader_config: Optional[Config] = None self._components: List[PipelineComponent] = [] self._selectors: List[Selector] = [] self._processors_index: Dict = {'': -1} self._configs: List[Optional[Config]] = [] # Will initialize at `initialize` because the processors length is # unknown. self._proc_mgr: ProcessManager = None # type: ignore self.evaluator_indices: List[int] = [] # needed for evaluator self._predict_to_gold: Dict[int, PackType] = {} if resource is None: self.resource = Resources() else: self.resource = resource self.initialized: bool = False self._check_type_consistency: bool = False
def __init__(self, resource: Optional[Resources] = None): self._reader: BaseReader self._reader_config: Optional[Config] self._components: List[PipelineComponent] = [] self._selectors: List[Selector] = [] self._processors_index: Dict = {'': -1} self._configs: List[Optional[Config]] = [] # Will intialize at `initialize` because the processors length is # unknown. self.proc_mgr: _ProcessManager # This manager controls global pack access information self._pack_manager: PackManager = PackManager() self._pack_manager.reset() self.evaluator_indices: List[int] = [] # needed for evaluator self._predict_to_gold: Dict[int, PackType] = {} if resource is None: self.resource = Resources() else: self.resource = resource self.initialized: bool = False
def __init__(self): self.resources: Resources = Resources() self.configs: Config = Config({}, {}) # Determine whether to check the consistencies between components. self._check_type_consistency: bool = False # The flag indicating whether the component is initialized. self.__is_initialized: bool = False
def __init__( self, train_reader: BaseReader, trainer: BaseTrainer, dev_reader: BaseReader, configs: Config, preprocessors: Optional[List[BaseProcessor]] = None, evaluator: Optional[Evaluator] = None, predictor: Optional[BaseProcessor] = None, ): self.resource = Resources() self.configs = configs train_reader.initialize(self.resource, self.configs.reader) if preprocessors is not None: for p in preprocessors: p.initialize(resources=self.resource, configs=configs.preprocessor) self.preprocessors = preprocessors else: self.preprocessors = [] self.train_reader = train_reader self.dev_reader = dev_reader self.trainer = trainer if predictor is not None: self.predictor = predictor if evaluator is not None: self.evaluator = evaluator self.evaluator.initialize(self.resource, self.configs.evaluator)
def test_save_without_keys(self): self.resources.save(output_dir=self.output_dir) new_resources = Resources() keys = list(self.kwargs.keys()) new_resources.load(keys=keys, path=self.output_dir) self.assertEqual(new_resources._resources, self.resources._resources)
def __init__(self, resource: Optional[Resources] = None): self._reader: BaseReader self._reader_config: Optional[HParams] self._processors: List[BaseProcessor] = [] self._selectors: List[Selector] = [] self._processors_index: Dict = {'': -1} self._configs: List[Optional[HParams]] = [] if resource is None: self.resource = Resources() else: self.resource = resource
def main(nif_context: str, nif_page_structure: str, mapping_literals: str, mapping_objects: str, nif_text_links: str, redirects: str, info_boxs_properties: str, base_output_path: str): # The datasets are read in a few steps. # 0. Load redirects between wikipedia pages. print_progress('Loading redirects', '\n') redirect_pickle = os.path.join(base_output_path, 'redirects.pickle') redirect_map: Dict[str, str] if os.path.exists(redirect_pickle): redirect_map = pickle.load(open(redirect_pickle, 'rb')) else: redirect_map = load_redirects(redirects) with open(redirect_pickle, 'wb') as pickle_f: pickle.dump(redirect_map, pickle_f) resources: Resources = Resources() resources.update(redirects=redirect_map) print_progress("Done loading.", '\n') # 1. Read the wiki text. raw_pack_dir = os.path.join(base_output_path, 'nif_raw') read_wiki_text(nif_context, raw_pack_dir, resources, True) print_progress("Done reading wikipedia text.", '\n') # 2. Add the rest of wiki page structures: struct_dir = raw_pack_dir + '_struct' add_wiki_info(WikiStructReader(), resources, nif_page_structure, raw_pack_dir, struct_dir, 'page_structures', True) print_progress("Done reading wikipedia structures.", '\n') link_dir = struct_dir + '_links' add_wiki_info(WikiAnchorReader(), resources, nif_text_links, struct_dir, link_dir, 'anchor_links', True) print_progress("Done reading wikipedia anchors.", '\n') property_dir = link_dir + '_property' add_wiki_info(WikiPropertyReader(), resources, info_boxs_properties, link_dir, property_dir, 'info_box_properties', True) print_progress("Done reading wikipedia info-boxes.", '\n') literal_dir = property_dir + '_literals' add_wiki_info(WikiInfoBoxReader(), resources, mapping_literals, property_dir, literal_dir, 'literals', True) print_progress("Done reading wikipedia info-boxes literals.", '\n') mapping_dir = literal_dir + '_objects' add_wiki_info(WikiInfoBoxReader(), resources, mapping_objects, literal_dir, mapping_dir, 'objects', True) print_progress("Done reading wikipedia info-boxes objects.", '\n')
def __init__(self, resource: Optional[Resources] = None): self._reader: BaseReader self._reader_config: Optional[HParams] self._processors: List[BaseProcessor] = [] self._selectors: List[Selector] = [] self._processors_index: Dict = {'': -1} self._configs: List[Optional[HParams]] = [] self._evaluator: Optional[Evaluator] = None self._evaluator_config: Optional[HParams] = None # needed for evaluator self._predict_to_gold: Dict[int, PackType] = {} if resource is None: self.resource = Resources() else: self.resource = resource
def __init__(self): self.resources: Resources = Resources() self.configs: Config = Config({}, {}) self._check_type_consistency = False
def setUp(self): self.kwargs = {"1": "one", "dummy": DummyObject(1, 2)} self.resources = Resources(**self.kwargs) self.output_dir = tempfile.mkdtemp()
def main(): config = yaml.safe_load(open("config.yml", "r")) config = HParams(config, default_hparams=None) if not os.path.exists(config.indexer.model_dir): print(f"Creating a new index...") encoder = BERTEncoder(pretrained_model_name="bert-base-uncased") encoder.to(device) feature_original_types = { "id": ["int64", "FixedLenFeature"], "input_ids": ["int64", "FixedLenFeature", config.indexer.max_seq_length], "segment_ids": ["int64", "FixedLenFeature", config.indexer.max_seq_length], "text": ["str", "FixedLenFeature"] } hparam = { "allow_smaller_final_batch": True, "batch_size": config.indexer.batch_size, "dataset": { "data_name": "data", "feature_original_types": feature_original_types, "files": config.indexer.pickle_data_dir }, "shuffle": False } print(f"Embedding the text using BERTEncoder...") record_data = RecordData(hparams=hparam, device=device) data_iterator = DataIterator(record_data) index = EmbeddingBasedIndexer(hparams={ "index_type": "GpuIndexFlatIP", "dim": 768, "device": "gpu0" }) for idx, batch in enumerate(data_iterator): ids = batch["id"] input_ids = batch["input_ids"] segment_ids = batch["segment_ids"] text = batch["text"] _, pooled_output = get_embeddings(encoder, input_ids, segment_ids) index.add(vectors=pooled_output, meta_data={k.item(): v for k, v in zip(ids, text)}) if (idx + 1) % 50 == 0: print(f"Completed {idx+1} batches of size " f"{config.indexer.batch_size}") index.save(path=config.indexer.model_dir) resource = Resources() query_pipeline = Pipeline(resource=resource) query_pipeline.set_reader(MultiPackTerminalReader()) query_pipeline.add_processor( processor=MachineTranslationProcessor(), config=config.translator) query_pipeline.add_processor( processor=QueryCreator(), config=config.query_creator) query_pipeline.add_processor( processor=SearchProcessor(), config=config.indexer) query_pipeline.add_processor( processor=NLTKSentenceSegmenter(), selector=NameMatchSelector(select_name="doc_0")) query_pipeline.add_processor( processor=NLTKWordTokenizer(), selector=NameMatchSelector(select_name="doc_0")) query_pipeline.add_processor( processor=NLTKPOSTagger(), selector=NameMatchSelector(select_name="doc_0")) query_pipeline.add_processor( processor=SRLPredictor(), config=config.SRL, selector=NameMatchSelector(select_name="doc_0")) # query_pipeline.add_processor( # processor=CoNLLNERPredictor(), config=config.NER, # selector=NameMatchSelector(select_name="doc_0")) query_pipeline.add_processor( processor=MachineTranslationProcessor(), config=config.back_translator) query_pipeline.initialize() for m_pack in query_pipeline.process_dataset(): # update resource to be used in the next conversation query_pack = m_pack.get_pack("query") if resource.get("user_utterance"): resource.get("user_utterance").append(query_pack) else: resource.update(user_utterance=[query_pack]) response_pack = m_pack.get_pack("response") if resource.get("bot_utterance"): resource.get("bot_utterance").append(response_pack) else: resource.update(bot_utterance=[response_pack]) english_pack = m_pack.get_pack("pack") print(colored("English Translation of the query: ", "green"), english_pack.text, "\n") pack = m_pack.get_pack("doc_0") print(colored("Retrieved Document", "green"), pack.text, "\n") print(colored("German Translation", "green"), m_pack.get_pack("response").text, "\n") for sentence in pack.get(Sentence): sent_text = sentence.text print(colored("Sentence:", 'red'), sent_text, "\n") print(colored("Semantic role labels:", 'red')) for link in pack.get(PredicateLink, sentence): parent = link.get_parent() child = link.get_child() print(f" - \"{child.text}\" is role {link.arg_type} of " f"predicate \"{parent.text}\"") print() input(colored("Press ENTER to continue...\n", 'green'))
def main(nif_context: str, nif_page_structure: str, mapping_literals: str, mapping_objects: str, nif_text_links: str, redirects: str, info_boxs_properties: str, base_output_path: str): # The datasets are read in a few steps. # 0. Load redirects between wikipedia pages. print_progress('Loading redirects', '\n') redirect_pickle = os.path.join(base_output_path, 'redirects.pickle') redirect_map: Dict[str, str] if os.path.exists(redirect_pickle): redirect_map = pickle.load(open(redirect_pickle, 'rb')) else: redirect_map = load_redirects(redirects) with open(redirect_pickle, 'wb') as pickle_f: pickle.dump(redirect_map, pickle_f) resources: Resources = Resources() resources.update(redirects=redirect_map) print_progress("Done loading.", '\n') # 1. Read the wiki text. raw_pack_dir = os.path.join(base_output_path, 'nif_raw') read_wiki_text(nif_context, raw_pack_dir, resources, True) print_progress("Done reading wikipedia text.", '\n') # 2. Add wiki page structures, create a new directory for it. struct_dir = raw_pack_dir + '_struct' add_wiki_info(WikiStructReader(), resources, nif_page_structure, raw_pack_dir, struct_dir, 'page_structures', True) print_progress("Done reading wikipedia structures.", '\n') # 3. Add wiki links, create a new directory for it. link_dir = struct_dir + '_links' add_wiki_info(WikiAnchorReader(), resources, nif_text_links, struct_dir, link_dir, 'anchor_links', True) print_progress("Done reading wikipedia anchors.", '\n') # 4 The following steps add info boxes: # 4.1 Add un-mapped infobox, we directly write to the previous directory property_dir = link_dir add_wiki_info(WikiPropertyReader(), resources, info_boxs_properties, link_dir, property_dir, 'info_box_properties', skip_existing=True, overwrite=True, output_index_file_name='properties.idx') print_progress("Done reading wikipedia info-boxes properties.", '\n') # 4.1 Add mapped literal, we directly write to the previous directory. literal_dir = property_dir add_wiki_info(WikiInfoBoxReader(), resources, mapping_literals, property_dir, literal_dir, 'literals', skip_existing=True, overwrite=True, output_index_file_name='literals.idx') print_progress("Done reading wikipedia info-boxes literals.", '\n') # 4.1 Add mapped object, we directly write to the previous directory. mapping_dir = literal_dir add_wiki_info(WikiInfoBoxReader(), resources, mapping_objects, literal_dir, mapping_dir, 'objects', skip_existing=True, overwrite=True, output_index_file_name='objects.idx') print_progress("Done reading wikipedia info-boxes objects.", '\n')
def main( nif_context: str, nif_page_structure: str, mapping_literals: str, mapping_objects: str, nif_text_links: str, redirects: str, info_boxs_properties: str, categories: str, base_output_path: str, resume_existing: bool, ): # Whether to skip the whole step. if resume_existing: skip_existing = False else: skip_existing = True # The datasets are read in a few steps. # 0. Load redirects between wikipedia pages. print_progress("Loading redirects", "\n") redirect_map: Dict[str, str] = cache_redirects(base_output_path, redirects) resources: Resources = Resources() resources.update(redirects=redirect_map) print_progress("Done loading.", "\n") # 1. Read the wiki text. raw_pack_dir = os.path.join(base_output_path, "nif_raw") read_wiki_text(nif_context, raw_pack_dir, resources, True) print_progress("Done reading wikipedia text.", "\n") # Use the same index structure for all writers. main_index = os.path.join(raw_pack_dir, "article.idx") # 2. Add wiki page structures, create a new directory for it. struct_dir = raw_pack_dir + "_struct" add_wiki_info( WikiStructReader(), resources, nif_page_structure, raw_pack_dir, struct_dir, "page_structures", use_input_index=True, skip_existing=skip_existing, resume_from_last=resume_existing, input_index_file_path=main_index, ) print_progress("Done reading wikipedia structures.", "\n") # 3. Add wiki links, create a new directory for it. link_dir = struct_dir + "_links" add_wiki_info( WikiAnchorReader(), resources, nif_text_links, struct_dir, link_dir, "anchor_links", use_input_index=True, skip_existing=True, resume_from_last=resume_existing, input_index_file_path=main_index, ) print_progress("Done reading wikipedia anchors.", "\n") # 4 The following steps add info boxes: # 4.1 Add un-mapped infobox, we directly write to the previous directory property_dir = link_dir add_wiki_info( WikiPropertyReader(), resources, info_boxs_properties, link_dir, property_dir, "info_box_properties", use_input_index=True, skip_existing=True, resume_from_last=resume_existing, output_index_file_name="properties.idx", input_index_file_path=main_index, ) print_progress("Done reading wikipedia info-boxes properties.", "\n") # 4.1 Add mapped literal, we directly write to the previous directory. literal_dir = property_dir add_wiki_info( WikiInfoBoxReader(), resources, mapping_literals, property_dir, literal_dir, "literals", use_input_index=True, skip_existing=True, resume_from_last=resume_existing, output_index_file_name="literals.idx", input_index_file_path=main_index, ) print_progress("Done reading wikipedia info-boxes literals.", "\n") # 4.1 Add mapped object, we directly write to the previous directory. mapping_dir = literal_dir add_wiki_info( WikiInfoBoxReader(), resources, mapping_objects, literal_dir, mapping_dir, "objects", use_input_index=True, skip_existing=True, resume_from_last=resume_existing, output_index_file_name="objects.idx", input_index_file_path=main_index, ) print_progress("Done reading wikipedia info-boxes objects.", "\n") # 4.2 Add category, directly write to previous directory. category_dir = mapping_dir add_wiki_info( WikiCategoryReader(), resources, categories, mapping_dir, category_dir, "categories", use_input_index=True, skip_existing=True, resume_from_last=resume_existing, output_index_file_name="categories.idx", input_index_file_path=main_index, )
def __init__( self, resource: Optional[Resources] = None, ontology_file: Optional[str] = None, enforce_consistency: bool = False, do_init_type_check: bool = False, ): r""" Args: resource: The ``Resources`` object, which is a global registry used in the pipeline. Objects defined as ``Resources`` will be passed on to the processors in the pipeline for initialization. ontology_file: The path to the input ontology specification file, which should be a json file, and it should have all the entries inside with no import as key. enforce_consistency: This boolean determines whether the pipeline will check the content expectations specified in each pipeline component. Each component will check whether the input pack contains the expected data via checking the meta-data, and throws a :class:`~forte.common.exception.ExpectedEntryNotFound` if it fails. When this function is called with enforce is ``True``, all the pipeline components would check if the input datapack record matches with the expected types and attributes if function ``expected_types_and_attributes`` is implemented for the processor. For example, processor A requires entry type of ``ft.onto.base_ontology.Sentence``, and processor B would produce this type in the output datapack, so ``record`` function of processor B writes the record of this type in the datapack and processor A implements ``expected_types_and_attributes`` to add this type. Then when the pipeline runs with `enforce_consistency=True`, processor A would check if this type exists in the record of the output of the previous pipeline component. do_init_type_check: Determine whether to check records types and attributes during pipeline initialization. Default to `False`. If this boolean is set to `True`, each component in the pipeline will be validated by comparing its ``expected_types_and_attributes`` with the accumulated ``records`` from all the downstream components. """ self._reader: BaseReader self._reader_config: Optional[Config] = None # These variables defines the units in the pipeline, they should be # of the same length self._components: List[PipelineComponent] = [] self._selectors: List[Selector] = [] self._configs: List[Optional[Config]] = [] # Maintain a set of the pipeline components to fast check whether # the component is already there. self.__component_set: Set[PipelineComponent] = set() # Will initialize at `initialize` because the processors length is # unknown. self._proc_mgr: ProcessManager = None # type: ignore self.evaluator_indices: List[int] = [] # needed for evaluator self._predict_to_gold: Dict[int, PackType] = {} if resource is None: self.resource = Resources() else: self.resource = resource if ontology_file is None: with resources.path("forte.ontology_specs", "base_ontology.json") as data_path: ontology_file = str(data_path) if ontology_file is not None: with open(ontology_file, "r") as f: spec_dict = json.load(f) self.resource.update(onto_specs_path=ontology_file) self.resource.update(onto_specs_dict=spec_dict) # The flag indicating whether this pipeline is initialized. self._initialized: bool = False # The flag indicating whether we want to enforce type consistency # between the processors. self._check_type_consistency: bool = False # Create one copy of the dummy selector to reduce class creation. self.__default_selector: Selector = DummySelector() # needed for time profiling of pipeline self._enable_profiling: bool = False self._profiler: List[float] = [] self._check_type_consistency = enforce_consistency # Indicate whether do type checking during pipeline initialization self._do_init_type_check: bool = do_init_type_check
def main(): config = yaml.safe_load(open("config.yml", "r")) config = HParams(config, default_hparams=None) resource = Resources() query_pipeline = Pipeline(resource=resource) query_pipeline.set_reader(reader=MultiPackTerminalReader(), config=config.reader) query_pipeline.add_processor(processor=MicrosoftBingTranslator(), config=config.translator) query_pipeline.add_processor(processor=BertBasedQueryCreator(), config=config.query_creator) query_pipeline.add_processor(processor=SearchProcessor(), config=config.indexer) query_pipeline.add_processor( processor=NLTKSentenceSegmenter(), selector=NameMatchSelector( select_name=config.indexer.response_pack_name[0])) query_pipeline.add_processor( processor=NLTKWordTokenizer(), selector=NameMatchSelector( select_name=config.indexer.response_pack_name[0])) query_pipeline.add_processor( processor=NLTKPOSTagger(), selector=NameMatchSelector( select_name=config.indexer.response_pack_name[0])) query_pipeline.add_processor( processor=SRLPredictor(), config=config.SRL, selector=NameMatchSelector( select_name=config.indexer.response_pack_name[0])) query_pipeline.add_processor(processor=MicrosoftBingTranslator(), config=config.back_translator) query_pipeline.initialize() for m_pack in query_pipeline.process_dataset(): # update resource to be used in the next conversation query_pack = m_pack.get_pack(config.translator.in_pack_name) if resource.get("user_utterance"): resource.get("user_utterance").append(query_pack) else: resource.update(user_utterance=[query_pack]) response_pack = m_pack.get_pack(config.back_translator.in_pack_name) if resource.get("bot_utterance"): resource.get("bot_utterance").append(response_pack) else: resource.update(bot_utterance=[response_pack]) english_pack = m_pack.get_pack("pack") print(colored("English Translation of the query: ", "green"), english_pack.text, "\n") pack = m_pack.get_pack(config.indexer.response_pack_name[0]) print(colored("Retrieved Document", "green"), pack.text, "\n") print(colored("German Translation", "green"), m_pack.get_pack("response").text, "\n") for sentence in pack.get(Sentence): sent_text = sentence.text print(colored("Sentence:", 'red'), sent_text, "\n") print(colored("Semantic role labels:", 'red')) for link in pack.get(PredicateLink, sentence): parent = link.get_parent() child = link.get_child() print(f" - \"{child.text}\" is role {link.arg_type} of " f"predicate \"{parent.text}\"") print() input(colored("Press ENTER to continue...\n", 'green'))