def test_run_translation_adapter(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" run_translation.py --model_name_or_path facebook/bart-base --source_lang en --target_lang ro --train_file ./tests/fixtures/tests_samples/wmt16/sample.json --validation_file ./tests/fixtures/tests_samples/wmt16/sample.json --output_dir {tmp_dir} --overwrite_output_dir --max_steps=50 --warmup_steps=8 --do_train --do_eval --learning_rate=3e-3 --per_device_train_batch_size=2 --per_device_eval_batch_size=1 --predict_with_generate --source_lang en_XX --target_lang ro_RO --train_adapter --adapter_config=houlsby --adapter_reduction_factor=8 """.split() with patch.object(sys, "argv", testargs): run_translation.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_bleu"], 30)
def test_run_clm_adapter(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" run_clm.py --model_name_or_path gpt2 --train_file ./tests/fixtures/sample_text.txt --validation_file ./tests/fixtures/sample_text.txt --do_train --do_eval --learning_rate 1e-3 --block_size 128 --per_device_train_batch_size 5 --per_device_eval_batch_size 5 --num_train_epochs 2 --output_dir {tmp_dir} --overwrite_output_dir --train_adapter --adapter_config=houlsby --adapter_reduction_factor=8 """.split() if torch.cuda.device_count() > 1: # Skipping because there are not enough batches to train the model + would need a drop_last to work. return if torch_device != "cuda": testargs.append("--no_cuda") with patch.object(sys, "argv", testargs): run_clm.main() result = get_results(tmp_dir) self.assertLess(result["perplexity"], 100)
def test_run_mlm_adapter(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" run_mlm.py --model_name_or_path roberta-base --train_file ./tests/fixtures/sample_text.txt --validation_file ./tests/fixtures/sample_text.txt --output_dir {tmp_dir} --overwrite_output_dir --do_train --do_eval --prediction_loss_only --num_train_epochs=1 --train_adapter --adapter_config=houlsby --adapter_reduction_factor=8 """.split() if torch_device != "cuda": testargs.append("--no_cuda") with patch.object(sys, "argv", testargs): run_mlm.main() result = get_results(tmp_dir) self.assertLess(result["perplexity"], 42)
def test_run_swag_adapter(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" run_swag.py --model_name_or_path bert-base-uncased --train_file ./tests/fixtures/tests_samples/swag/sample.json --validation_file ./tests/fixtures/tests_samples/swag/sample.json --output_dir {tmp_dir} --overwrite_output_dir --max_steps=20 --warmup_steps=2 --do_train --do_eval --learning_rate=2e-4 --per_device_train_batch_size=2 --per_device_eval_batch_size=1 --train_adapter --adapter_config=houlsby --adapter_reduction_factor=8 """.split() with patch.object(sys, "argv", testargs): run_swag.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.8)
def test_run_ner_adapter(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu epochs = 14 if get_gpu_count() > 1 else 6 tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" run_ner.py --model_name_or_path bert-base-uncased --train_file ./tests/fixtures/tests_samples/conll/sample.json --validation_file ./tests/fixtures/tests_samples/conll/sample.json --output_dir {tmp_dir} --overwrite_output_dir --do_train --do_eval --warmup_steps=2 --learning_rate=5e-3 --per_device_train_batch_size=2 --per_device_eval_batch_size=2 --num_train_epochs={epochs} --train_adapter --adapter_config=houlsby --adapter_reduction_factor=16 """.split() if torch_device != "cuda": testargs.append("--no_cuda") with patch.object(sys, "argv", testargs): run_ner.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) self.assertGreaterEqual(result["eval_precision"], 0.75) self.assertLess(result["eval_loss"], 0.5)