def main():
import json
from pathlib import Path
import sys
sys.path += ["."]
import examples.question_answering.qa_sparse_xp as qa_sparse_xp
if len(sys.argv) < 2:
raise RuntimeError("Please specify json file")
filename = Path(sys.argv[1]).resolve()
param_dict = json.load(open(filename))
# variants = [{"dense_lambda":1.0, "regularization_final_lambda": 20}]
# for variant in variants:
# param_dict_ = copy.deepcopy(param_dict)
# param_dict_.update(variant)
qa = qa_sparse_xp.QASparseXP(param_dict)
def hp_space(trial):
return {}
qa.hyperparameter_search(direction="minimize",
hp_space=hp_space,
n_trials=1)
def main():
import json
from pathlib import Path
if len(sys.argv) < 2:
raise RuntimeError("Please specify json file")
filename = Path(sys.argv[1]).resolve()
param_dict = json.load(open(filename))
import examples.question_answering.qa_sparse_xp as qa_sparse_xp
qa = qa_sparse_xp.QASparseXP(param_dict)
qa.run()
def finetune(ctx, task, json_path, model_name_or_path, teacher,
per_device_train_batch_size, regularization_final_lambda,
dense_lambda, ampere_pruning_method, output_dir, layer_norm_patch,
gelu_patch):
filename = json_path
if json_path is not None:
param_dict = json.load(open(filename))
else:
if task == "squadv1":
param_dict = QA_TYPICAL_PARAMETERS
elif task == "mnli":
param_dict = MNLI_TYPICAL_PARAMETERS
else:
raise ValueError(f"Unknown task {task}")
param_dict["model_name_or_path"] = model_name_or_path
param_dict["distil_teacher_name_or_path"] = teacher
param_dict["per_device_train_batch_size"] = per_device_train_batch_size
param_dict["regularization_final_lambda"] = regularization_final_lambda
param_dict["dense_lambda"] = dense_lambda
param_dict["ampere_pruning_method"] = ampere_pruning_method
param_dict["output_dir"] = output_dir
param_dict["logging_dir"] = output_dir
param_dict["layer_norm_patch"] = layer_norm_patch
param_dict["gelu_patch"] = gelu_patch
if task == "squadv1":
if json_path is None and teacher is None:
# Large teacher is default
param_dict[
"distil_teacher_name_or_path"] = "csarron/bert-base-uncased-squad-v1"
import examples.question_answering.qa_sparse_xp as qa_sparse_xp
experiment = qa_sparse_xp.QASparseXP(param_dict)
else:
if json_path is None and teacher is None or teacher == "bert-large-uncased-whole-word-masking-finetuned-squad":
# Large teacher is default
param_dict[
"distil_teacher_name_or_path"] = "aloxatel/bert-base-mnli"
import examples.text_classification.glue_sparse_xp as glue_sparse_xp
experiment = glue_sparse_xp.GlueSparseXP(param_dict)
# This does not actually use hyper parameter search right now, but it's useful for naming the output directory for example
experiment.hyperparameter_search()
def finetune(
ctx,
task,
json_path,
model_name_or_path,
teacher,
per_device_train_batch_size,
regularization_final_lambda,
dense_lambda,
ampere_pruning_method,
output_dir,
layer_norm_patch,
gelu_patch
):
if json_path is not None:
param_dict = json.load(open(json_path))
else:
if task in QA_TASKS:
param_dict = QA_TYPICAL_PARAMETERS
if task == "squadv2":
param_dict["dataset_name"] = "squad_v2"
param_dict["version_2_with_negative"] = 1
else:
param_dict["dataset_name"] = "squad"
param_dict["version_2_with_negative"] = 0
elif task in GLUE_TASKS:
param_dict = GLUE_TYPICAL_PARAMETERS
param_dict["dataset_name"] = task
else:
raise ValueError(f"Unknown task {task}")
param_dict["model_name_or_path"] = model_name_or_path
param_dict["distil_teacher_name_or_path"] = teacher
param_dict["per_device_train_batch_size"] = per_device_train_batch_size
param_dict["regularization_final_lambda"] = regularization_final_lambda
param_dict["dense_lambda"] = dense_lambda
param_dict["ampere_pruning_method"] = ampere_pruning_method
param_dict["output_dir"] = output_dir
param_dict["logging_dir"] = output_dir
param_dict["layer_norm_patch"] = layer_norm_patch
param_dict["gelu_patch"] = gelu_patch
if teacher == "auto":
name_teacher = task
if model_name_or_path == "bert-base-uncased":
name_teacher += '_base'
elif model_name_or_path == "bert-large-uncased":
name_teacher += '_large'
distil_teacher_name_or_path = task2teacher.get(name_teacher)
if distil_teacher_name_or_path is None:
raise ValueError(f"Cannot find teacher for model {model_name_or_path} on task {task}")
param_dict["distil_teacher_name_or_path"] = distil_teacher_name_or_path
if task in QA_TASKS:
import examples.question_answering.qa_sparse_xp as qa_sparse_xp
experiment = qa_sparse_xp.QASparseXP(param_dict)
else:
import examples.text_classification.glue_sparse_xp as glue_sparse_xp
experiment = glue_sparse_xp.GlueSparseXP(param_dict)
# This does not actually use hyper parameter search right now, but it's useful for naming the output directory for example
experiment.hyperparameter_search()