def setup(config: Config) -> Pipeline:
resource = Resources()
query_pipeline = Pipeline[MultiPack](resource=resource)
query_pipeline.set_reader(
reader=MultiPackTerminalReader(), config=config.reader)
query_pipeline.add(
component=MicrosoftBingTranslator(), config=config.translator)
query_pipeline.add(
component=BertBasedQueryCreator(), config=config.query_creator)
query_pipeline.add(
component=SearchProcessor(), config=config.searcher)
top_response_pack_name = config.indexer.response_pack_name + '_0'
query_pipeline.add(
component=NLTKSentenceSegmenter(),
selector=NameMatchSelector(select_name=top_response_pack_name))
query_pipeline.add(
component=NLTKWordTokenizer(),
selector=NameMatchSelector(select_name=top_response_pack_name))
query_pipeline.add(
component=NLTKPOSTagger(),
selector=NameMatchSelector(select_name=top_response_pack_name))
query_pipeline.add(
component=SRLPredictor(), config=config.SRL,
selector=NameMatchSelector(select_name=top_response_pack_name))
query_pipeline.add(
component=MicrosoftBingTranslator(), config=config.back_translator)
query_pipeline.initialize()
return query_pipeline
def setUp(self):
self.kwargs = {
'1': 'one',
'dummy': DummyObject(1, 2)
}
self.resources = Resources(**self.kwargs)
self.output_dir = tempfile.mkdtemp()
def __init__(self, resource: Optional[Resources] = None):
self._reader: BaseReader
self._reader_config: Optional[Config] = None
self._components: List[PipelineComponent] = []
self._selectors: List[Selector] = []
self._processors_index: Dict = {'': -1}
self._configs: List[Optional[Config]] = []
# Will initialize at `initialize` because the processors length is
# unknown.
self._proc_mgr: ProcessManager = None # type: ignore
self.evaluator_indices: List[int] = []
# needed for evaluator
self._predict_to_gold: Dict[int, PackType] = {}
if resource is None:
self.resource = Resources()
else:
self.resource = resource
self.initialized: bool = False
self._check_type_consistency: bool = False
def __init__(self, resource: Optional[Resources] = None):
self._reader: BaseReader
self._reader_config: Optional[Config]
self._components: List[PipelineComponent] = []
self._selectors: List[Selector] = []
self._processors_index: Dict = {'': -1}
self._configs: List[Optional[Config]] = []
# Will intialize at `initialize` because the processors length is
# unknown.
self.proc_mgr: _ProcessManager
# This manager controls global pack access information
self._pack_manager: PackManager = PackManager()
self._pack_manager.reset()
self.evaluator_indices: List[int] = []
# needed for evaluator
self._predict_to_gold: Dict[int, PackType] = {}
if resource is None:
self.resource = Resources()
else:
self.resource = resource
self.initialized: bool = False
def __init__(self):
self.resources: Resources = Resources()
self.configs: Config = Config({}, {})
# Determine whether to check the consistencies between components.
self._check_type_consistency: bool = False
# The flag indicating whether the component is initialized.
self.__is_initialized: bool = False
def __init__(
self,
train_reader: BaseReader,
trainer: BaseTrainer,
dev_reader: BaseReader,
configs: Config,
preprocessors: Optional[List[BaseProcessor]] = None,
evaluator: Optional[Evaluator] = None,
predictor: Optional[BaseProcessor] = None,
):
self.resource = Resources()
self.configs = configs
train_reader.initialize(self.resource, self.configs.reader)
if preprocessors is not None:
for p in preprocessors:
p.initialize(resources=self.resource,
configs=configs.preprocessor)
self.preprocessors = preprocessors
else:
self.preprocessors = []
self.train_reader = train_reader
self.dev_reader = dev_reader
self.trainer = trainer
if predictor is not None:
self.predictor = predictor
if evaluator is not None:
self.evaluator = evaluator
self.evaluator.initialize(self.resource, self.configs.evaluator)
def test_save_without_keys(self):
self.resources.save(output_dir=self.output_dir)
new_resources = Resources()
keys = list(self.kwargs.keys())
new_resources.load(keys=keys, path=self.output_dir)
self.assertEqual(new_resources._resources, self.resources._resources)
def __init__(self, resource: Optional[Resources] = None):
self._reader: BaseReader
self._reader_config: Optional[HParams]
self._processors: List[BaseProcessor] = []
self._selectors: List[Selector] = []
self._processors_index: Dict = {'': -1}
self._configs: List[Optional[HParams]] = []
if resource is None:
self.resource = Resources()
else:
self.resource = resource
def main(nif_context: str, nif_page_structure: str, mapping_literals: str,
mapping_objects: str, nif_text_links: str, redirects: str,
info_boxs_properties: str, base_output_path: str):
# The datasets are read in a few steps.
# 0. Load redirects between wikipedia pages.
print_progress('Loading redirects', '\n')
redirect_pickle = os.path.join(base_output_path, 'redirects.pickle')
redirect_map: Dict[str, str]
if os.path.exists(redirect_pickle):
redirect_map = pickle.load(open(redirect_pickle, 'rb'))
else:
redirect_map = load_redirects(redirects)
with open(redirect_pickle, 'wb') as pickle_f:
pickle.dump(redirect_map, pickle_f)
resources: Resources = Resources()
resources.update(redirects=redirect_map)
print_progress("Done loading.", '\n')
# 1. Read the wiki text.
raw_pack_dir = os.path.join(base_output_path, 'nif_raw')
read_wiki_text(nif_context, raw_pack_dir, resources, True)
print_progress("Done reading wikipedia text.", '\n')
# 2. Add the rest of wiki page structures:
struct_dir = raw_pack_dir + '_struct'
add_wiki_info(WikiStructReader(), resources, nif_page_structure,
raw_pack_dir, struct_dir, 'page_structures', True)
print_progress("Done reading wikipedia structures.", '\n')
link_dir = struct_dir + '_links'
add_wiki_info(WikiAnchorReader(), resources, nif_text_links, struct_dir,
link_dir, 'anchor_links', True)
print_progress("Done reading wikipedia anchors.", '\n')
property_dir = link_dir + '_property'
add_wiki_info(WikiPropertyReader(), resources, info_boxs_properties,
link_dir, property_dir, 'info_box_properties', True)
print_progress("Done reading wikipedia info-boxes.", '\n')
literal_dir = property_dir + '_literals'
add_wiki_info(WikiInfoBoxReader(), resources, mapping_literals,
property_dir, literal_dir, 'literals', True)
print_progress("Done reading wikipedia info-boxes literals.", '\n')
mapping_dir = literal_dir + '_objects'
add_wiki_info(WikiInfoBoxReader(), resources, mapping_objects, literal_dir,
mapping_dir, 'objects', True)
print_progress("Done reading wikipedia info-boxes objects.", '\n')
def __init__(self, resource: Optional[Resources] = None):
self._reader: BaseReader
self._reader_config: Optional[HParams]
self._processors: List[BaseProcessor] = []
self._selectors: List[Selector] = []
self._processors_index: Dict = {'': -1}
self._configs: List[Optional[HParams]] = []
self._evaluator: Optional[Evaluator] = None
self._evaluator_config: Optional[HParams] = None
# needed for evaluator
self._predict_to_gold: Dict[int, PackType] = {}
if resource is None:
self.resource = Resources()
else:
self.resource = resource
def __init__(self):
self.resources: Resources = Resources()
self.configs: Config = Config({}, {})
self._check_type_consistency = False
def setUp(self):
self.kwargs = {"1": "one", "dummy": DummyObject(1, 2)}
self.resources = Resources(**self.kwargs)
self.output_dir = tempfile.mkdtemp()
def main():
config = yaml.safe_load(open("config.yml", "r"))
config = HParams(config, default_hparams=None)
if not os.path.exists(config.indexer.model_dir):
print(f"Creating a new index...")
encoder = BERTEncoder(pretrained_model_name="bert-base-uncased")
encoder.to(device)
feature_original_types = {
"id": ["int64", "FixedLenFeature"],
"input_ids": ["int64", "FixedLenFeature",
config.indexer.max_seq_length],
"segment_ids": ["int64", "FixedLenFeature",
config.indexer.max_seq_length],
"text": ["str", "FixedLenFeature"]
}
hparam = {
"allow_smaller_final_batch": True,
"batch_size": config.indexer.batch_size,
"dataset": {
"data_name": "data",
"feature_original_types": feature_original_types,
"files": config.indexer.pickle_data_dir
},
"shuffle": False
}
print(f"Embedding the text using BERTEncoder...")
record_data = RecordData(hparams=hparam, device=device)
data_iterator = DataIterator(record_data)
index = EmbeddingBasedIndexer(hparams={
"index_type": "GpuIndexFlatIP",
"dim": 768,
"device": "gpu0"
})
for idx, batch in enumerate(data_iterator):
ids = batch["id"]
input_ids = batch["input_ids"]
segment_ids = batch["segment_ids"]
text = batch["text"]
_, pooled_output = get_embeddings(encoder, input_ids, segment_ids)
index.add(vectors=pooled_output,
meta_data={k.item(): v for k, v in zip(ids, text)})
if (idx + 1) % 50 == 0:
print(f"Completed {idx+1} batches of size "
f"{config.indexer.batch_size}")
index.save(path=config.indexer.model_dir)
resource = Resources()
query_pipeline = Pipeline(resource=resource)
query_pipeline.set_reader(MultiPackTerminalReader())
query_pipeline.add_processor(
processor=MachineTranslationProcessor(), config=config.translator)
query_pipeline.add_processor(
processor=QueryCreator(), config=config.query_creator)
query_pipeline.add_processor(
processor=SearchProcessor(), config=config.indexer)
query_pipeline.add_processor(
processor=NLTKSentenceSegmenter(),
selector=NameMatchSelector(select_name="doc_0"))
query_pipeline.add_processor(
processor=NLTKWordTokenizer(),
selector=NameMatchSelector(select_name="doc_0"))
query_pipeline.add_processor(
processor=NLTKPOSTagger(),
selector=NameMatchSelector(select_name="doc_0"))
query_pipeline.add_processor(
processor=SRLPredictor(), config=config.SRL,
selector=NameMatchSelector(select_name="doc_0"))
# query_pipeline.add_processor(
# processor=CoNLLNERPredictor(), config=config.NER,
# selector=NameMatchSelector(select_name="doc_0"))
query_pipeline.add_processor(
processor=MachineTranslationProcessor(), config=config.back_translator)
query_pipeline.initialize()
for m_pack in query_pipeline.process_dataset():
# update resource to be used in the next conversation
query_pack = m_pack.get_pack("query")
if resource.get("user_utterance"):
resource.get("user_utterance").append(query_pack)
else:
resource.update(user_utterance=[query_pack])
response_pack = m_pack.get_pack("response")
if resource.get("bot_utterance"):
resource.get("bot_utterance").append(response_pack)
else:
resource.update(bot_utterance=[response_pack])
english_pack = m_pack.get_pack("pack")
print(colored("English Translation of the query: ", "green"),
english_pack.text, "\n")
pack = m_pack.get_pack("doc_0")
print(colored("Retrieved Document", "green"), pack.text, "\n")
print(colored("German Translation", "green"),
m_pack.get_pack("response").text, "\n")
for sentence in pack.get(Sentence):
sent_text = sentence.text
print(colored("Sentence:", 'red'), sent_text, "\n")
print(colored("Semantic role labels:", 'red'))
for link in pack.get(PredicateLink, sentence):
parent = link.get_parent()
child = link.get_child()
print(f" - \"{child.text}\" is role {link.arg_type} of "
f"predicate \"{parent.text}\"")
print()
input(colored("Press ENTER to continue...\n", 'green'))
def main(nif_context: str, nif_page_structure: str, mapping_literals: str,
mapping_objects: str, nif_text_links: str, redirects: str,
info_boxs_properties: str, base_output_path: str):
# The datasets are read in a few steps.
# 0. Load redirects between wikipedia pages.
print_progress('Loading redirects', '\n')
redirect_pickle = os.path.join(base_output_path, 'redirects.pickle')
redirect_map: Dict[str, str]
if os.path.exists(redirect_pickle):
redirect_map = pickle.load(open(redirect_pickle, 'rb'))
else:
redirect_map = load_redirects(redirects)
with open(redirect_pickle, 'wb') as pickle_f:
pickle.dump(redirect_map, pickle_f)
resources: Resources = Resources()
resources.update(redirects=redirect_map)
print_progress("Done loading.", '\n')
# 1. Read the wiki text.
raw_pack_dir = os.path.join(base_output_path, 'nif_raw')
read_wiki_text(nif_context, raw_pack_dir, resources, True)
print_progress("Done reading wikipedia text.", '\n')
# 2. Add wiki page structures, create a new directory for it.
struct_dir = raw_pack_dir + '_struct'
add_wiki_info(WikiStructReader(), resources, nif_page_structure,
raw_pack_dir, struct_dir, 'page_structures', True)
print_progress("Done reading wikipedia structures.", '\n')
# 3. Add wiki links, create a new directory for it.
link_dir = struct_dir + '_links'
add_wiki_info(WikiAnchorReader(), resources, nif_text_links, struct_dir,
link_dir, 'anchor_links', True)
print_progress("Done reading wikipedia anchors.", '\n')
# 4 The following steps add info boxes:
# 4.1 Add un-mapped infobox, we directly write to the previous directory
property_dir = link_dir
add_wiki_info(WikiPropertyReader(),
resources,
info_boxs_properties,
link_dir,
property_dir,
'info_box_properties',
skip_existing=True,
overwrite=True,
output_index_file_name='properties.idx')
print_progress("Done reading wikipedia info-boxes properties.", '\n')
# 4.1 Add mapped literal, we directly write to the previous directory.
literal_dir = property_dir
add_wiki_info(WikiInfoBoxReader(),
resources,
mapping_literals,
property_dir,
literal_dir,
'literals',
skip_existing=True,
overwrite=True,
output_index_file_name='literals.idx')
print_progress("Done reading wikipedia info-boxes literals.", '\n')
# 4.1 Add mapped object, we directly write to the previous directory.
mapping_dir = literal_dir
add_wiki_info(WikiInfoBoxReader(),
resources,
mapping_objects,
literal_dir,
mapping_dir,
'objects',
skip_existing=True,
overwrite=True,
output_index_file_name='objects.idx')
print_progress("Done reading wikipedia info-boxes objects.", '\n')
def main(
nif_context: str,
nif_page_structure: str,
mapping_literals: str,
mapping_objects: str,
nif_text_links: str,
redirects: str,
info_boxs_properties: str,
categories: str,
base_output_path: str,
resume_existing: bool,
):
# Whether to skip the whole step.
if resume_existing:
skip_existing = False
else:
skip_existing = True
# The datasets are read in a few steps.
# 0. Load redirects between wikipedia pages.
print_progress("Loading redirects", "\n")
redirect_map: Dict[str, str] = cache_redirects(base_output_path, redirects)
resources: Resources = Resources()
resources.update(redirects=redirect_map)
print_progress("Done loading.", "\n")
# 1. Read the wiki text.
raw_pack_dir = os.path.join(base_output_path, "nif_raw")
read_wiki_text(nif_context, raw_pack_dir, resources, True)
print_progress("Done reading wikipedia text.", "\n")
# Use the same index structure for all writers.
main_index = os.path.join(raw_pack_dir, "article.idx")
# 2. Add wiki page structures, create a new directory for it.
struct_dir = raw_pack_dir + "_struct"
add_wiki_info(
WikiStructReader(),
resources,
nif_page_structure,
raw_pack_dir,
struct_dir,
"page_structures",
use_input_index=True,
skip_existing=skip_existing,
resume_from_last=resume_existing,
input_index_file_path=main_index,
)
print_progress("Done reading wikipedia structures.", "\n")
# 3. Add wiki links, create a new directory for it.
link_dir = struct_dir + "_links"
add_wiki_info(
WikiAnchorReader(),
resources,
nif_text_links,
struct_dir,
link_dir,
"anchor_links",
use_input_index=True,
skip_existing=True,
resume_from_last=resume_existing,
input_index_file_path=main_index,
)
print_progress("Done reading wikipedia anchors.", "\n")
# 4 The following steps add info boxes:
# 4.1 Add un-mapped infobox, we directly write to the previous directory
property_dir = link_dir
add_wiki_info(
WikiPropertyReader(),
resources,
info_boxs_properties,
link_dir,
property_dir,
"info_box_properties",
use_input_index=True,
skip_existing=True,
resume_from_last=resume_existing,
output_index_file_name="properties.idx",
input_index_file_path=main_index,
)
print_progress("Done reading wikipedia info-boxes properties.", "\n")
# 4.1 Add mapped literal, we directly write to the previous directory.
literal_dir = property_dir
add_wiki_info(
WikiInfoBoxReader(),
resources,
mapping_literals,
property_dir,
literal_dir,
"literals",
use_input_index=True,
skip_existing=True,
resume_from_last=resume_existing,
output_index_file_name="literals.idx",
input_index_file_path=main_index,
)
print_progress("Done reading wikipedia info-boxes literals.", "\n")
# 4.1 Add mapped object, we directly write to the previous directory.
mapping_dir = literal_dir
add_wiki_info(
WikiInfoBoxReader(),
resources,
mapping_objects,
literal_dir,
mapping_dir,
"objects",
use_input_index=True,
skip_existing=True,
resume_from_last=resume_existing,
output_index_file_name="objects.idx",
input_index_file_path=main_index,
)
print_progress("Done reading wikipedia info-boxes objects.", "\n")
# 4.2 Add category, directly write to previous directory.
category_dir = mapping_dir
add_wiki_info(
WikiCategoryReader(),
resources,
categories,
mapping_dir,
category_dir,
"categories",
use_input_index=True,
skip_existing=True,
resume_from_last=resume_existing,
output_index_file_name="categories.idx",
input_index_file_path=main_index,
)
def __init__(
self,
resource: Optional[Resources] = None,
ontology_file: Optional[str] = None,
enforce_consistency: bool = False,
do_init_type_check: bool = False,
):
r"""
Args:
resource: The ``Resources`` object, which is a global registry used
in the pipeline. Objects defined as ``Resources`` will be
passed on to the processors in the
pipeline for initialization.
ontology_file: The path to the input ontology specification file,
which should be a json file, and it should have all the entries
inside with no import as key.
enforce_consistency: This boolean determines whether the
pipeline will check the content expectations specified in each
pipeline component. Each component will check whether the input
pack contains the expected data
via checking the meta-data, and throws a
:class:`~forte.common.exception.ExpectedEntryNotFound` if it
fails. When this function is called with enforce is ``True``,
all the pipeline components would check if the input datapack
record matches
with the expected types and attributes if function
``expected_types_and_attributes`` is implemented
for the processor. For example, processor A requires entry type
of ``ft.onto.base_ontology.Sentence``, and processor B would
produce this type in the output datapack, so ``record`` function
of processor B writes the record of this type in the datapack
and processor A implements ``expected_types_and_attributes`` to
add this type. Then when the pipeline runs with
`enforce_consistency=True`, processor A would check if this
type exists in the record of the output of the
previous pipeline component.
do_init_type_check: Determine whether to check records types and
attributes during pipeline initialization. Default to `False`.
If this boolean is set to `True`, each component in the
pipeline will be validated by comparing its
``expected_types_and_attributes`` with the accumulated
``records`` from all the downstream components.
"""
self._reader: BaseReader
self._reader_config: Optional[Config] = None
# These variables defines the units in the pipeline, they should be
# of the same length
self._components: List[PipelineComponent] = []
self._selectors: List[Selector] = []
self._configs: List[Optional[Config]] = []
# Maintain a set of the pipeline components to fast check whether
# the component is already there.
self.__component_set: Set[PipelineComponent] = set()
# Will initialize at `initialize` because the processors length is
# unknown.
self._proc_mgr: ProcessManager = None # type: ignore
self.evaluator_indices: List[int] = []
# needed for evaluator
self._predict_to_gold: Dict[int, PackType] = {}
if resource is None:
self.resource = Resources()
else:
self.resource = resource
if ontology_file is None:
with resources.path("forte.ontology_specs",
"base_ontology.json") as data_path:
ontology_file = str(data_path)
if ontology_file is not None:
with open(ontology_file, "r") as f:
spec_dict = json.load(f)
self.resource.update(onto_specs_path=ontology_file)
self.resource.update(onto_specs_dict=spec_dict)
# The flag indicating whether this pipeline is initialized.
self._initialized: bool = False
# The flag indicating whether we want to enforce type consistency
# between the processors.
self._check_type_consistency: bool = False
# Create one copy of the dummy selector to reduce class creation.
self.__default_selector: Selector = DummySelector()
# needed for time profiling of pipeline
self._enable_profiling: bool = False
self._profiler: List[float] = []
self._check_type_consistency = enforce_consistency
# Indicate whether do type checking during pipeline initialization
self._do_init_type_check: bool = do_init_type_check
def main():
config = yaml.safe_load(open("config.yml", "r"))
config = HParams(config, default_hparams=None)
resource = Resources()
query_pipeline = Pipeline(resource=resource)
query_pipeline.set_reader(reader=MultiPackTerminalReader(),
config=config.reader)
query_pipeline.add_processor(processor=MicrosoftBingTranslator(),
config=config.translator)
query_pipeline.add_processor(processor=BertBasedQueryCreator(),
config=config.query_creator)
query_pipeline.add_processor(processor=SearchProcessor(),
config=config.indexer)
query_pipeline.add_processor(
processor=NLTKSentenceSegmenter(),
selector=NameMatchSelector(
select_name=config.indexer.response_pack_name[0]))
query_pipeline.add_processor(
processor=NLTKWordTokenizer(),
selector=NameMatchSelector(
select_name=config.indexer.response_pack_name[0]))
query_pipeline.add_processor(
processor=NLTKPOSTagger(),
selector=NameMatchSelector(
select_name=config.indexer.response_pack_name[0]))
query_pipeline.add_processor(
processor=SRLPredictor(),
config=config.SRL,
selector=NameMatchSelector(
select_name=config.indexer.response_pack_name[0]))
query_pipeline.add_processor(processor=MicrosoftBingTranslator(),
config=config.back_translator)
query_pipeline.initialize()
for m_pack in query_pipeline.process_dataset():
# update resource to be used in the next conversation
query_pack = m_pack.get_pack(config.translator.in_pack_name)
if resource.get("user_utterance"):
resource.get("user_utterance").append(query_pack)
else:
resource.update(user_utterance=[query_pack])
response_pack = m_pack.get_pack(config.back_translator.in_pack_name)
if resource.get("bot_utterance"):
resource.get("bot_utterance").append(response_pack)
else:
resource.update(bot_utterance=[response_pack])
english_pack = m_pack.get_pack("pack")
print(colored("English Translation of the query: ", "green"),
english_pack.text, "\n")
pack = m_pack.get_pack(config.indexer.response_pack_name[0])
print(colored("Retrieved Document", "green"), pack.text, "\n")
print(colored("German Translation", "green"),
m_pack.get_pack("response").text, "\n")
for sentence in pack.get(Sentence):
sent_text = sentence.text
print(colored("Sentence:", 'red'), sent_text, "\n")
print(colored("Semantic role labels:", 'red'))
for link in pack.get(PredicateLink, sentence):
parent = link.get_parent()
child = link.get_child()
print(f" - \"{child.text}\" is role {link.arg_type} of "
f"predicate \"{parent.text}\"")
print()
input(colored("Press ENTER to continue...\n", 'green'))