Python Config.add_hparam 예제들

예제 #1

파일: main_train.py 프로젝트: jennyzhang-petuum/forte

def main():
    config_data = yaml.safe_load(open("config_data.yml", "r"))
    config_model = yaml.safe_load(open("config_model.yml", "r"))
    config_evaluator = yaml.safe_load(open("config_evaluator.yml", "r"))
    config_preprocess = yaml.safe_load(open("config_preprocessor.yml", "r"))

    # All the configs
    config = Config({}, default_hparams=None)
    config.add_hparam("config_data", config_data)
    config.add_hparam("config_model", config_model)
    config.add_hparam("preprocessor", config_preprocess)
    config.add_hparam("reader", {})
    config.add_hparam("evaluator", config_evaluator)

    reader = CoNLL03Reader()

    # Keep the vocabulary processor as a simple counter
    vocab_processor = CoNLL03VocabularyProcessor()

    ner_trainer = CoNLLNERTrainer()
    ner_predictor = CoNLLNERPredictor()
    ner_evaluator = CoNLLNEREvaluator()

    train_pipe = TrainPipeline(
        train_reader=reader,
        trainer=ner_trainer,
        dev_reader=reader,
        configs=config,
        preprocessors=[vocab_processor],
        predictor=ner_predictor,
        evaluator=ner_evaluator,
    )
    train_pipe.run()

예제 #2

파일: main_predict.py 프로젝트: gxchris95/forte-1

# limitations under the License.

import yaml

from forte.common.configuration import Config
from forte.data.data_pack import DataPack
from forte.pipeline import Pipeline
from forte.data.readers.conll03_reader import CoNLL03Reader
from forte.processors.ner_predictor import CoNLLNERPredictor
from ft.onto.base_ontology import Token, Sentence, EntityMention

config_data = yaml.safe_load(open("config_data.yml", "r"))
config_model = yaml.safe_load(open("config_model.yml", "r"))

config = Config({}, default_hparams=None)
config.add_hparam('config_data', config_data)
config.add_hparam('config_model', config_model)

pl = Pipeline[DataPack]()
pl.set_reader(CoNLL03Reader())
pl.add(CoNLLNERPredictor(), config=config)

pl.initialize()

for pack in pl.process_dataset(config.config_data.test_path):
    for pred_sentence in pack.get_data(
            context_type=Sentence,
            request={
                Token: {
                    "fields": ["ner"]
                },

예제 #3

파일: main_predict.py 프로젝트: jennyzhang-petuum/forte

# limitations under the License.

import yaml

from forte.common.configuration import Config
from forte.data.data_pack import DataPack
from forte.pipeline import Pipeline
from forte.data.readers.conll03_reader import CoNLL03Reader
from forte.processors.nlp import CoNLLNERPredictor
from ft.onto.base_ontology import Token, Sentence, EntityMention

config_data = yaml.safe_load(open("config_data.yml", "r"))
config_model = yaml.safe_load(open("config_model.yml", "r"))

config = Config({}, default_hparams=None)
config.add_hparam("config_data", config_data)
config.add_hparam("config_model", config_model)

pl = Pipeline[DataPack]()
pl.set_reader(CoNLL03Reader())
pl.add(CoNLLNERPredictor(), config=config)

pl.initialize()

for pack in pl.process_dataset(config.config_data.test_path):
    for pred_sentence in pack.get_data(
        context_type=Sentence,
        request={
            Token: {"fields": ["ner"]},
            Sentence: [],  # span by default
            EntityMention: {},