def default_trainer_params(cls):
p = super().default_trainer_params()
p.gen.train = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "conll_txt_val_small.lst")])
p.gen.val = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "conll_txt_val_small.lst")])
p.scenario.data.tags = Resource(
workdir_path(__file__, "data", "tags", "tags_conll.txt"))
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p.gen.train = ListsFileGeneratorParams(lists=[
workdir_path(__file__, "lists", "conll_json_val_small.lst")
])
p.gen.val = ListsFileGeneratorParams(lists=[
workdir_path(__file__, "lists", "conll_json_val_small.lst")
])
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p = set_test_trainer_params(p)
p.gen.train = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "dewebcrawl_debug.lst")])
p.gen.val = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "dewebcrawl_debug.lst")])
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p = set_test_trainer_params(p)
p.gen.train = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "ler_debug.lst")])
p.gen.val = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "ler_debug.lst")])
p.scenario.data.tags = Resource(
workdir_path(__file__, "data", "tags", "ler_fg.txt"))
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p.scenario.data.segment_train = True
p.gen.train = ListsFileGeneratorParams(lists=[
workdir_path(__file__, "lists", "dewebcrawl_seg_debug.lst")
])
p.gen.val = ListsFileGeneratorParams(lists=[
workdir_path(__file__, "lists", "dewebcrawl_seg_debug.lst")
])
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p = set_test_trainer_params(p)
p.gen.train = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "paifile_small.lst")])
p.gen.val = ListsFileGeneratorParams(
lists=[workdir_path(__file__, "lists", "paifile_small.lst")])
p.scenario.data.tags = Resource(
workdir_path(__file__, "data", "tags", "paifile_tags.txt"))
p.scenario.data.paifile_input = True
return p
def default_trainer_params(cls):
p = super().default_trainer_params()
p = set_test_trainer_params(p)
p.scenario.data.tags = Resource(
workdir_path(__file__, "data", "tags", "tags_germeval_14.txt"))
# p.gen = FromDatasetsTrainerGeneratorParams
# p.gen.train = ListsFileGeneratorParams(lists=[workdir_path(__file__, 'lists', 'ler_debug.lst')])
# p.gen.val = ListsFileGeneratorParams(lists=[workdir_path(__file__, 'lists', 'ler_debug.lst')])
return p
def set_test_trainer_params(p):
p.random_seed = 123
p.gen.setup.train.batch_size = 2
p.gen.setup.train.prefetch = 1
p.gen.setup.train.num_processes = 1
p.gen.setup.val.batch_size = 2
p.gen.setup.val.prefetch = 1
p.gen.setup.val.num_processes = 1
data = p.scenario.data
data.tokenizer = Resource(workdir_path(__file__, "data", "tokenizer", "tokenizer_de.subwords"))
model = p.scenario.model
model.d_model = 2
model.dff = 2
model.num_layers = 1
model.num_heads = 2
return p
def setUp(self) -> None:
os.chdir(workdir_path(__file__))