def table_exp():
params = parse_args('test')
for ds in ['mini', 'tiered']:
for shot in [1, 5]:
params.dataset = ds
params.n_shot = shot
run_exp(params, verbose=500)
def unbalanced_exp():
params = parse_args('test')
for ds in ['mini', 'tiered']:
params.dataset = ds
exp_dict = {}
for q in [0, 10, 20, 30, 40, 50]:
params.n_unbalance_max = q
res, ci = run_exp(params, verbose=500)
exp_dict[q] = (res, ci)
with open(f'exp_unbalanced_{ds}.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
def n_query_exp():
params = parse_args('test')
for ds in ['mini', 'tiered']:
params.dataset = ds
exp_dict = {}
for q in [2, 5, 10, 15, 30, 40, 50]:
params.n_query = q
res, ci = run_exp(params, verbose=1000)
exp_dict[q] = (res, ci)
with open(f'exp_n_queries_{ds}.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
def table_exp():
params = parse_args('test')
exp_dict = {}
for ds in ['mini', 'tiered']:
for shot in [1, 5]:
params.dataset = ds
params.n_shot = shot
res, ci = run_exp(params, verbose=500)
exp_dict[f'{ds} {shot} shot'] = (res, ci)
with open('exp_table.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
def noise_exp():
params = parse_args('test')
params.n_semi = 100
for ds in ['mini', 'tiered']:
params.dataset = ds
exp_dict = {}
for q in range(0, 7+1):
params.n_distract = q
res, ci = run_exp(params, verbose=500)
exp_dict[q] = (res, ci)
with open(f'exp_noise_semi_{ds}.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
def semi_exp():
params = parse_args('test')
exp_dict = {}
for ds in ['mini', 'tiered']:
for shot in [1, 5]:
for num_semi in [30, 50, 100]:
params.dataset = ds
params.n_shot = shot
params.n_semi = num_semi
res, ci = run_exp(params, verbose=500)
exp_dict[f'{ds} {shot} shot {num_semi} unlabeled'] = (res, ci)
with open('exp_semi.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
def projection_dim_exp():
args = [5, 10, 15, 20]
params = parse_args('test')
params.n_shot = 1
for ds in ['mini', 'tiered']:
params.dataset = ds
exp_dict = {'args': args}
for part in ['val', 'test']:
params.part = part
res, ci = run_exp(params, verbose=500, args=args)
exp_dict[f'{part}'] = (res, ci)
with open(f'exp_projection_dim_{ds}.pickle', 'wb') as handle:
pickle.dump(exp_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
import sys
import math
import torch
import logging
import os
from tqdm import tqdm
from typing import Dict
from transformers.trainer_utils import EvalPrediction
from transformers.integrations import TensorBoardCallback
import json
import sys
if __name__ == '__main__':
# read config file and merge with arguments
args = parse_args()
# path to saved models (based on BERT/CharacterBERT)
checkpoints = {
"charbert_xs_open_uf": "/data/darwin/xs_open_uf_2e-5/checkpoint-13818",
"charbert_xs_open_ua": "/data/darwin/xs_open_ua_2e-5/checkpoint-8430",
"charbert_xl_open_uf": "/data/darwin/xl_open_uf_2e-5/checkpoint-23023",
"charbert_xl_open_ua": "/data/darwin/xl_open_ua_2e-5/checkpoint-7770",
"bert": "pretrained_models/bert-base-uncased",
"bert_fs10": "/pan2020/fewshot_chkps/reddit_fewshotbase10%overall0.682",
"bert_fs20": "/pan2020/fewshot_chkps/reddit_fewshotbase20%overall0.672",
"bert_fs50": "/pan2020/fewshot_chkps/reddit_fewshotbase50%overall0.78",
"bert_fs100": "/pan2020/fewshot_chkps/reddit_fewshotbase100%overall0.865",
"bert_xl_closed_v1": "/pan2020/pretrained_models/bert-base-uncased-closed-v1-xlloss0.1975",
"bert_xl_closed_v2": "/pan2020/pretrained_models/bert-base-uncased-closed-v2-xlloss0.2693",
"bert_xl_open_ua": "/pan2020/pretrained_models/bert-base-uncased-open-unseenauthors-xlloss0.4251",