def test_bert_sklearn_accy(): """ Test bert_sklearn accuracy compare against huggingface run_classifier.py on 200 rows of SST-2 data. """ print("Running bert-sklearn...") X_train, y_train, X_dev, y_dev, label_list = toxic_test_data() # define model model = BertClassifier() model.validation_fraction = 0.0 model.learning_rate = 5e-5 model.gradient_accumulation_steps = 2 model.max_seq_length = 64 model.train_batch_size = 16 model.eval_batch_size = 8 model.epochs = 2 model.multilabel = True # for multi-label classification model.label_list = label_list model.fit(X_train, y_train) bert_sklearn_accy = model.score(X_dev, y_dev) bert_sklearn_accy /= 100 # run huggingface BERT run_classifier and check we get the same accuracy cmd = r"python tests/run_classifier.py --task_name sst-2 \ --data_dir ./tests/data/sst2 \ --do_train --do_eval \ --output_dir ./comptest \ --bert_model bert-base-uncased \ --do_lower_case \ --learning_rate 5e-5 \ --gradient_accumulation_steps 2 \ --max_seq_length 64 \ --train_batch_size 16 \ --eval_batch_size 8 \ --num_train_epochs 2" print("\nRunning huggingface run_classifier.py...\n") os.system(cmd) print("...finished run_classifier.py\n") # parse run_classifier.py output file and find the accy accy = open("comptest/eval_results.txt").read().split("\n")[ 0] # 'acc = 0.76' accy = accy.split("=")[1] accy = float(accy) print("bert_sklearn accy: %.02f, run_classifier.py accy : %0.02f" % (bert_sklearn_accy, accy)) # clean up print("\nCleaning up eval file: eval_results.txt") #os.remove("eval_results.txt") shutil.rmtree("comptest") assert bert_sklearn_accy == accy
def bert_model(): model = BertClassifier() # model.bert_model = 'bert-base-uncased' model.bert_model = 'bert-large-uncased' # model.bert_model = 'scibert-basevocab-uncased' # model.num_mlp_layers = 10 model.max_seq_length = 64 model.epochs = 4 # model.learning_rate = 4e-5 model.learning_rate = 2e-5 model.gradient_accumulation_steps = 1 return model
data_df_not_na_label = data_df_not_na[data_df_not_na['label'].isin( ['0', '1', '-1'])] data_df_not_na_label['label'].value_counts() train_df, dev_df = train_test_split(data_df_not_na_label, test_size=0.2, shuffle=True) ## 准备模型的数据 X_train, y_train = train_df['content'], train_df['label'] X_dev, y_dev = dev_df['content'], dev_df['label'] # define model model = BertClassifier('bert-base-uncased') model.validation_fraction = 0.0 model.learning_rate = 3e-5 model.gradient_accumulation_steps = 1 model.max_seq_length = 64 model.train_batch_size = 1 model.eval_batch_size = 1 model.epochs = 1 # fit model.fit(X_train, y_train) # score accy = model.score(X_dev, y_dev) test_df = pd.read_csv( 'data/nCov_10k_test.csv', skiprows=[0],