def test_doc_classification(caplog):
caplog.set_level(logging.CRITICAL)
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=False)
n_epochs = 1
batch_size = 8
evaluate_every = 30
lang_model = "bert-base-german-cased"
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=lang_model, do_lower_case=False)
processor = GermEval18CoarseProcessor(tokenizer=tokenizer,
max_seq_len=64,
data_dir="samples/doc_class",
train_filename="train-sample.tsv",
test_filename=None)
data_silo = DataSilo(processor=processor, batch_size=batch_size)
language_model = Bert.load(lang_model)
prediction_head = TextClassificationHead(
layer_dims=[768, len(processor.label_list)])
model = AdaptiveModel(language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_sequence"],
device=device)
optimizer, warmup_linear = initialize_optimizer(
model=model,
learning_rate=2e-5,
warmup_proportion=0.1,
n_examples=data_silo.n_samples("train"),
batch_size=batch_size,
n_epochs=1)
trainer = Trainer(optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
warmup_linear=warmup_linear,
evaluate_every=evaluate_every,
device=device)
model = trainer.train(model)
save_dir = "testsave/doc_class"
model.save(save_dir)
processor.save(save_dir)
basic_texts = [
{
"text":
"Schartau sagte dem Tagesspiegel, dass Fischer ein Idiot sei"
},
{
"text": "Martin Müller spielt Handball in Berlin"
},
]
model = Inferencer.load(save_dir)
result = model.run_inference(dicts=basic_texts)
assert result[0]["predictions"][0]["label"] == "OTHER"
assert abs(result[0]["predictions"][0]["probability"] -
0.5358161) <= 0.0001
def run_experiment(args):
validate_args(args)
directory_setup(output_dir=args.output_dir, do_train=args.do_train)
distributed = bool(args.local_rank != -1)
# Init device and distributed settings
device, n_gpu = initialize_device_settings(use_cuda=args.cuda,
local_rank=args.local_rank,
fp16=args.fp16)
args.batch_size = args.batch_size // args.gradient_accumulation_steps
if n_gpu > 1:
args.batch_size = args.batch_size * n_gpu
set_all_seeds(args.seed)
# Prepare Data
tokenizer = BertTokenizer.from_pretrained(args.model,
do_lower_case=args.lower_case)
processor = Processor.load(
processor_name=args.processor_name,
tokenizer=tokenizer,
max_seq_len=args.max_seq_len,
data_dir=args.data_dir,
)
data_silo = DataSilo(processor=processor,
batch_size=args.batch_size,
distributed=distributed)
class_weights = None
if args.balance_classes:
class_weights = data_silo.class_weights
model = get_adaptive_model(
lm_output_type=args.lm_output_type,
prediction_heads=args.prediction_head,
layer_dims=args.layer_dims,
model=args.model,
device=device,
class_weights=class_weights,
fp16=args.fp16,
embeds_dropout_prob=args.embeds_dropout_prob,
local_rank=args.local_rank,
n_gpu=n_gpu,
)
# Init optimizer
# TODO: warmup linear is sometimes NONE depending on fp16 - is there a neater way to handle this?
optimizer, warmup_linear = initialize_optimizer(
model=model,
learning_rate=args.learning_rate,
warmup_proportion=args.warmup_proportion,
loss_scale=args.loss_scale,
fp16=args.fp16,
n_examples=data_silo.n_samples("train"),
batch_size=args.batch_size,
grad_acc_steps=args.gradient_accumulation_steps,
n_epochs=args.epochs,
)
trainer = Trainer(
optimizer=optimizer,
data_silo=data_silo,
epochs=args.epochs,
n_gpu=n_gpu,
grad_acc_steps=args.gradient_accumulation_steps,
fp16=args.fp16,
warmup_linear=warmup_linear,
evaluate_every=args.eval_every,
device=device,
)
model = trainer.train(model)
model_name = (
f"{model.language_model.name}-{model.language_model.language}-{args.name}"
)
processor.save(f"saved_models/{model_name}")
model.save(f"saved_models/{model_name}")
loss_ignore_index=-1)
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[lm_prediction_head, next_sentence_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_token", "per_sequence"],
device=device,
)
# 5. Create an optimizer
optimizer, warmup_linear = initialize_optimizer(
model=model,
learning_rate=2e-5,
warmup_proportion=0.1,
n_examples=data_silo.n_samples("train"),
batch_size=16,
n_epochs=1,
)
# 6. Feed everything to the Trainer, which keeps care of growing our model into powerful plant and evaluates it from time to time
trainer = Trainer(
optimizer=optimizer,
data_silo=data_silo,
epochs=10,
n_gpu=1,
warmup_linear=warmup_linear,
evaluate_every=100,
device=device,
)
def test_qa(caplog):
caplog.set_level(logging.CRITICAL)
set_all_seeds(seed=42)
device, n_gpu = initialize_device_settings(use_cuda=False)
batch_size = 6
n_epochs = 2
evaluate_every = 100
base_LM_model = "bert-base-cased"
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=base_LM_model, do_lower_case=False)
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=64,
train_filename="train-sample.json",
dev_filename="dev-sample.json",
test_filename=None,
data_dir="samples/qa",
)
data_silo = DataSilo(processor=processor,
batch_size=batch_size,
distributed=False)
language_model = Bert.load(base_LM_model)
prediction_head = QuestionAnsweringHead(
layer_dims=[768, len(processor.label_list)])
model = AdaptiveModel(
language_model=language_model,
prediction_heads=[prediction_head],
embeds_dropout_prob=0.1,
lm_output_types=["per_token"],
device=device,
)
optimizer, warmup_linear = initialize_optimizer(
model=model,
learning_rate=1e-5,
warmup_proportion=0.2,
n_examples=data_silo.n_samples("train"),
batch_size=batch_size,
n_epochs=n_epochs,
)
trainer = Trainer(
optimizer=optimizer,
data_silo=data_silo,
epochs=n_epochs,
n_gpu=n_gpu,
warmup_linear=warmup_linear,
evaluate_every=evaluate_every,
device=device,
)
model = trainer.train(model)
save_dir = "testsave/qa"
model.save(save_dir)
processor.save(save_dir)
QA_input = [{
"questions": ["In what country is Normandy located?"],
"text":
"The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries."
}]
model = Inferencer(save_dir)
result = model.run_inference(dicts=QA_input)
assert result[0]["predictions"][0]["label"] == 'The'
