def test_fidelity():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
is_senn = False
e_config = ExperimentConfig()
e_config.name = "NLIEx_{}".format("Fidelity")
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
if is_senn:
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
else:
e_config.load_names = ['bert', 'cls_dense']
explain_tag = 'conflict'
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLIEx_Y_conflict", 'model-12039')
#load_id = ("NLI_Only_C", 'model-0')
#e.eval_fidelity(nli_setting, e_config, data_loader, load_id, explain_tag)
e.eval_fidelity_gradient(nli_setting, e_config, data_loader, load_id,
explain_tag)
def train_snli_ex():
hp = hyperparams.HPBert()
hp.compare_deletion_num = 20
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "SNLIEx_B"
e_config.ex_val = False
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict']
#explain_tag = 'match' # 'dontcare' 'match' 'mismatch'
#explain_tag = 'mismatch'
#explain_tag = 'conflict'
data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename,
True)
#load_id = ("NLI_run_nli_warm", "model-97332")
#load_id = ("NLIEx_A", "model-16910")
#load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt')
#load_id = ("NLIEx_D", "model-1964")
#load_id = ("NLIEx_D", "model-1317")
load_id = ("SNLI_Only_A", 'model-0')
e.train_nli_any_way(nli_setting, e_config, data_loader, load_id)
def analyze_nli_ex():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
explain_tag = 'match'
e_config = ExperimentConfig()
#e_config.name = "NLIEx_{}_premade_analyze".format(explain_tag)
e_config.name = "NLIEx_{}_analyze".format(explain_tag)
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#load_id = ("NLIEx_E_align", "model-23621")
#load_id = ("NLIEx_I_match", "model-1238")
if explain_tag == 'conflict':
load_id = ("NLIEx_Y_conflict", "model-12039")
#load_id = ("NLIEx_HB", "model-2684")
elif explain_tag == 'match':
load_id = ("NLIEx_P_match", "model-1636")
load_id = ("NLIEx_X_match", "model-12238")
elif explain_tag == 'mismatch':
load_id = ("NLIEx_U_mismatch", "model-10265")
e.nli_visualization(nli_setting, e_config, data_loader, load_id,
explain_tag)
def train_nli_smart_rf():
hp = hyperparams.HPSENLI()
hp.compare_deletion_num = 20
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
#explain_tag = 'mismatch'
explain_tag = 'match'
#explain_tag = 'mismatch'
loss_type = 2
e_config.name = "NLIEx_Hinge_{}".format(explain_tag)
e_config.num_epoch = 1
e_config.ex_val = True
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict']
e_config.save_eval = True
e_config.save_name = "LossFn_{}_{}".format(loss_type, explain_tag)
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLI_run_A", 'model-0')
print("Loss : ", loss_type)
e.train_nli_smart(nli_setting, e_config, data_loader, load_id, explain_tag,
loss_type)
def predict_rf():
hp = hyperparams.HPBert()
hp.batch_size = 256
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
target_label = 'mismatch'
#data_id = 'test_conflict'
data_id = "{}_1000".format(target_label)
e_config = ExperimentConfig()
#del_g = 0.7
#e_config.name = "X_match_del_{}".format(del_g)
e_config.name = "NLIEx_AnyA_{}".format(target_label)
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#load_id = ("NLI_bare_A", 'model-195608')
#load_id = ("NLIEx_O", 'model-10278')
load_id = ("NLIEx_W_mismatch", "model-12030")
load_id = ("NLIEx_Y_conflict", "model-12039")
load_id = ("NLIEx_X_match", "model-12238")
#load_id = ("NLIEx_match_del_{}".format(del_g), "model-4390")
load_id = ("NLIEx_CE_{}".format(target_label), "model-12199")
load_id = ("NLIEx_AnyA", "model-7255")
e.predict_rf(nli_setting, e_config, data_loader, load_id, data_id, 5)
def ubuntu_train_gen():
hp = hyperparams.HPUbuntu()
e = Experiment(hp)
voca_setting = NLI()
voca_setting.vocab_size = 30522
voca_setting.vocab_filename = "bert_voca.txt"
data_loader = ubuntu.DataLoader(hp.seq_max, voca_setting.vocab_filename,
voca_setting.vocab_size, True)
for i in range(3, 30):
ubuntu.batch_encode(i)
def test_ubuntu():
hp = hyperparams.HPUbuntu()
hp.batch_size = 16
e = Experiment(hp)
voca_setting = NLI()
voca_setting.vocab_size = 30522
voca_setting.vocab_filename = "bert_voca.txt"
data_loader = ubuntu.DataLoader(hp.seq_max, voca_setting.vocab_filename,
voca_setting.vocab_size, True)
e.test_valid_ubuntu(data_loader)
def train_nil():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_only_{}".format("B")
e_config.num_epoch = 2
e_config.save_interval = 30 * 60 # 30 minutes
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
e.train_nli(nli_setting, e_config, data_loader)
def pred_snli_ex():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "SNLIEx_B"
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename,
True)
load_id = ("SNLIEx_B", 'model-10275')
e.predict_rf(nli_setting, e_config, data_loader, load_id, "test")
def pred_mnli_anyway():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_AnyA"
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
target_label = 'mismatch'
data_id = "{}_1000".format(target_label)
load_id = ("NLIEx_AnyA", 'model-2785')
e.predict_rf(nli_setting, e_config, data_loader, load_id, data_id)
def train_nil():
hp = HP()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_only_{}".format("512")
e_config.num_epoch = 2
e_config.save_interval = 30 * 60 # 30 minutes
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt')
e.train_nli_ex_0(nli_setting, e_config, data_loader, load_id, False)
def train_nil_cold():
print('train_nil_cold')
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_Cold"
e_config.num_epoch = 2
e_config.save_interval = 30 * 60 # 30 minutes
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
saved = e.train_nli_ex_0(nli_setting, e_config, data_loader, None, False)
e.test_acc2(nli_setting, e_config, data_loader, saved)
def run_nli_w_path(run_name, step_name, model_path):
#run_name
hp = HPBert()
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
data_loader = nli.DataLoader(hp.seq_max, "bert_voca.txt", True)
data = get_nli_batches_from_data_loader(data_loader, hp.batch_size)
run_name = "{}_{}_NLI".format(run_name, step_name)
saved_model = train_nli(hp, nli_setting, run_name, 3, data, model_path)
tf.reset_default_graph()
avg_acc = test_nli(hp, nli_setting, run_name, data, saved_model)
print("avg_acc: ", avg_acc)
save_report("nli", run_name, step_name, avg_acc)
def train_nli_with_premade(explain_tag):
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_{}".format("Premade_"+explain_tag)
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert'] #, 'cls_dense'] #, 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt')
e.train_nli_ex_with_premade_data(nli_setting, e_config, data_loader, load_id, explain_tag)
def analyze_nli_pair():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_pair_analyze"
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLIEx_T", "model-12097")
e.nli_visualization_pairing(nli_setting, e_config, data_loader, load_id,
data)
def train_nil_on_bert():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_Only_{}".format("C")
e_config.num_epoch = 2
e_config.save_interval = 30 * 60 # 30 minutes
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#load_id = None
load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt')
#load_id = ("NLI_bert_w_explain", 'model-91531')
#load_id = ("NLI_Only_A", "model-0")
e.train_nli_ex_0(nli_setting, e_config, data_loader, load_id, False)
def test_nli():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_400k_tlm_simple_wo_hint"
e_config.num_epoch = 2
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] # , 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#saved = "/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_Cold/model-0"
saved = "/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_400k_tlm_wo_hint/model-0"
saved = '/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_400k_tlm_simple_hint/model-0'
print(saved)
e.test_acc2(nli_setting, e_config, data_loader, saved)
def interactive():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIInterative"
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLI_Only_B", 'model-0')
e.nli_interactive_list(nli_setting, e_config, data_loader, load_id)
def attribution_explain():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_run_{}".format("nli_eval")
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLI_run_nli_warm", "model-97332")
load_id = ("NLI_Only_A", 'model-0')
e.nli_attribution_baselines(nli_setting, e_config, data_loader, load_id)
def predict_lime_snli_continue():
hp = hyperparams.HPBert()
hp.batch_size = 512 + 256
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "SNLI_LIME_{}".format("eval")
e_config.load_names = ['bert', 'cls_dense']
data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename,
True)
load_id = ("SNLI_Only_A", 'model-0')
e.predict_lime_snli_continue(nli_setting, e_config, data_loader, load_id,
"test")
def do_test_dev_acc():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_Test"
e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#load_id = ("NLI_bare_A", 'model-195608')
load_id = ("NLIEx_S", 'model-4417')
load_id = ("NLIEx_Y_conflict", "model-9636")
load_id = ("NLI_Only_C", 'model-0')
e.test_acc(nli_setting, e_config, data_loader, load_id)
def interactive_visual():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIInterative"
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
load_id = ("NLIEx_U_mismatch", "model-10265")
load_id = ("NLIEx_Y_conflict", 'model-12039')
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
e.nli_interactive_visual(nli_setting, e_config, data_loader, load_id)
def train_mnli_any_way():
hp = HP()
hp.batch_size = 8
hp.compare_deletion_num = 20
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_Any_512"
e_config.ex_val = False
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] # , 'aux_conflict']
e_config.v2_load = True
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("nli512", 'model.ckpt-65000')
e.train_nli_any_way(nli_setting, e_config, data_loader, load_id)
def train_ukp_with_nli(load_id, exp_name):
step_per_epoch = 24544 + 970
hp = hyperparams.HPBert()
e_config = ExperimentConfig()
e_config.num_steps = step_per_epoch
e_config.save_interval = 100 * 60 # 30 minutes
e_config.voca_size = 30522
e_config.num_dev_batches = 30
e_config.load_names = ['bert']
e_config.valid_freq = 500
encode_opt = "is_good"
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
num_class_list = [3, 3]
f1_list = []
for topic in data_generator.argmining.ukp_header.all_topics:
e = Experiment(hp)
print(exp_name)
e_config.name = "argmix_{}_{}_{}".format(exp_name, topic, encode_opt)
arg_data_loader = BertDataLoader(topic,
True,
hp.seq_max,
"bert_voca.txt",
option=encode_opt)
nli_data_loader = nli.DataLoader(hp.seq_max,
nli_setting.vocab_filename, True)
shared_data_loader = SharedFeeder([arg_data_loader, nli_data_loader],
[1, 5], ["Arg", "NLI"],
hp.batch_size)
save_path = e.train_shared(e_config, shared_data_loader,
num_class_list, load_id)
print(topic)
f1_last = e.eval_ukp_on_shared(e_config, arg_data_loader,
num_class_list, save_path)
f1_list.append((topic, f1_last))
print(exp_name)
print(encode_opt)
print(f1_list)
for key, score in f1_list:
print("{0}\t{1:.03f}".format(key, score))
def visualize_senli_on_plain_text():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_lm_analyze"
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense', 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLIEx_AnyA", "model-7255")
p = "/mnt/scratch/youngwookim/Chair/data/tf/1"
data = list(extract_stream(p))
e.nli_visualization_lm(nli_setting, e_config, data_loader, load_id, data)
def train_pairing():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLIEx_{}".format("T")
e_config.num_epoch = 1
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense'] # 'aux_conflict']
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
#load_id = ("NLIEx_P_match", "model-1636")
load_id = ("NLI_run_A", 'model-0')
PAIRING_NLI = 6
e.train_nli_smart(nli_setting, e_config, data_loader, load_id, 'match',
PAIRING_NLI)
def train_snli_on_bert():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "SNLI_Only_{}".format("1")
e_config.num_epoch = 1
e_config.save_interval = 3 * 60 * 60 # 30 minutes
data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename,
True)
#load_id = None
load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt')
e.train_nli_ex_0(nli_setting,
e_config,
data_loader,
load_id,
f_train_ex=False)
def test_snli():
hp = hyperparams.HPBert()
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "SNLIEx_Test"
e_config.load_names = ['bert', 'cls_dense'] # , 'aux_conflict']
data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename,
True)
todo = []
load_id = ("SNLI_Only_A", 'model-0')
todo.append(load_id)
todo.append(("SNLI_Only_1", 'model-0'))
for load_id in todo:
tf.reset_default_graph()
e.test_acc(nli_setting, e_config, data_loader, load_id)
def baseline_explain():
hp = hyperparams.HPBert()
hp.batch_size *= 32
e = Experiment(hp)
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_run_{}".format("nli_warm")
e_config.num_epoch = 4
e_config.save_interval = 30 * 60 # 30 minutes
e_config.load_names = ['bert', 'cls_dense']
explain_tag = 'conflict' # 'dontcare' 'match' 'mismatch'
#explain_tag = 'match'
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLI_run_A", 'model-0')
e.nli_explain_baselines(nli_setting, e_config, data_loader, load_id,
explain_tag)
def attribution_predict():
hp = hyperparams.HPBert()
target_label = 'mismatch'
e = Experiment(hp)
hp.batch_size = 512
nli_setting = NLI()
nli_setting.vocab_size = 30522
nli_setting.vocab_filename = "bert_voca.txt"
e_config = ExperimentConfig()
e_config.name = "NLI_run_{}".format("nli_eval")
e_config.load_names = ['bert', 'cls_dense']
#data_id = "test_{}".format(target_label)
data_id = "{}_1000".format(target_label)
data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True)
load_id = ("NLI_Only_C", 'model-0')
e.nli_attribution_predict(nli_setting, e_config, data_loader, load_id,
target_label, data_id)