def hyperargs(): # type: () -> {}
"""
Builds different sets of arguments for the classifier. Must be the same for
training and predicting.
:return: the labeled arguments
:rtype: {}
"""
retdict = {}
for curwindow in [128, 64, 32, 256]:
for curstride in [0.7, 0.8, 0.9]:
accargs = ClassificationArgs()
accargs.num_train_epochs = 5
accargs.fp16 = False
accargs.overwrite_output_dir = True
accargs.evaluate_during_training = False
accargs.sliding_window = True
accargs.max_seq_length = curwindow
accargs.stride = curstride
accargs.labels_list = [1, 0]
accargs.save_eval_checkpoints = False
accargs.save_model_every_epoch = False
accargs.silent = True
accargs.manual_seed = 18
retdict['basic5epochs' + str(curwindow) + 'win' +
str(int(curstride * 10.0)) + 'stride'] = accargs
return retdict
def buildbertargs(): # type: () -> ClassificationArgs
"""
Builds arguments for the classifier. Must be the same for
training and predicting.
:return: the arguments
:rtype: ClassificationArgs
"""
accargs = ClassificationArgs()
accargs.num_train_epochs = 5
accargs.fp16 = False
accargs.overwrite_output_dir = True
accargs.evaluate_during_training = False
accargs.sliding_window = True
accargs.max_seq_length = 256
accargs.stride = 0.9
accargs.labels_list = [1, 0]
accargs.save_model_every_epoch = False
accargs.silent = True
accargs.manual_seed = 18
return accargs
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)
# Preparing train data
train_df = load_rte_data_file("data/train.jsonl")
eval_df = pd.read_json("data/eval_df", lines=True, orient="records")
test_df = pd.read_json("data/test_df", lines=True, orient="records")
model_args = ClassificationArgs()
model_args.eval_batch_size = 8
model_args.evaluate_during_training = True
model_args.evaluate_during_training_silent = False
model_args.evaluate_during_training_steps = 1000
model_args.learning_rate = 4e-4
model_args.manual_seed = 4
model_args.max_seq_length = 256
model_args.multiprocessing_chunksize = 5000
model_args.no_cache = True
model_args.no_save = True
model_args.num_train_epochs = 10
model_args.overwrite_output_dir = True
model_args.reprocess_input_data = True
model_args.train_batch_size = 16
model_args.gradient_accumulation_steps = 2
model_args.train_custom_parameters_only = False
model_args.labels_list = ["not_entailment", "entailment"]
model_args.wandb_project = "RTE - Hyperparameter Optimization"
def train():
# Initialize a new wandb run