def train_test_repeat(load_id, exp_name, n_repeat): hp = hyperparams.HPBert() e_config = ExperimentConfig() e_config.name = "RTE_{}".format("A") e_config.num_epoch = 10 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert'] vocab_filename = "bert_voca.txt" data_loader = rte.DataLoader(hp.seq_max, vocab_filename, True) print(load_id) scores = [] for i in range(n_repeat): e = Experiment(hp) print(exp_name) e_config.name = "rte_{}".format(exp_name) save_path = e.train_rte(e_config, data_loader, load_id) acc = e.eval_rte(e_config, data_loader, save_path) scores.append(acc) print(exp_name) for e in scores: print(e, end="\t") print() r = average(scores) print("Avg\n{0:.03f}".format(r)) return r
def test_fidelity(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" is_senn = False e_config = ExperimentConfig() e_config.name = "NLIEx_{}".format("Fidelity") e_config.num_epoch = 4 e_config.save_interval = 30 * 60 # 30 minutes if is_senn: e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] else: e_config.load_names = ['bert', 'cls_dense'] explain_tag = 'conflict' data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("NLIEx_Y_conflict", 'model-12039') #load_id = ("NLI_Only_C", 'model-0') #e.eval_fidelity(nli_setting, e_config, data_loader, load_id, explain_tag) e.eval_fidelity_gradient(nli_setting, e_config, data_loader, load_id, explain_tag)
def ukp_train_test_repeat(load_id, exp_name, topic, n_repeat): hp = hyperparams.HPBert() e_config = ExperimentConfig() e_config.num_epoch = 2 e_config.save_interval = 100 * 60 # 30 minutes e_config.voca_size = 30522 e_config.load_names = ['bert'] encode_opt = "is_good" print(load_id) scores = [] for i in range(n_repeat): e = Experiment(hp) print(exp_name) e_config.name = "arg_{}_{}_{}".format(exp_name, topic, encode_opt) data_loader = BertDataLoader(topic, True, hp.seq_max, "bert_voca.txt", option=encode_opt) save_path = e.train_ukp(e_config, data_loader, load_id) f1_last = e.eval_ukp(e_config, data_loader, save_path) scores.append(f1_last) print(exp_name) print(encode_opt) for e in scores: print(e, end="\t") print() print("Avg\n{0:.03f}".format(average(scores)))
def analyze_nli_ex(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" explain_tag = 'match' e_config = ExperimentConfig() #e_config.name = "NLIEx_{}_premade_analyze".format(explain_tag) e_config.name = "NLIEx_{}_analyze".format(explain_tag) e_config.num_epoch = 4 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) #load_id = ("NLIEx_E_align", "model-23621") #load_id = ("NLIEx_I_match", "model-1238") if explain_tag == 'conflict': load_id = ("NLIEx_Y_conflict", "model-12039") #load_id = ("NLIEx_HB", "model-2684") elif explain_tag == 'match': load_id = ("NLIEx_P_match", "model-1636") load_id = ("NLIEx_X_match", "model-12238") elif explain_tag == 'mismatch': load_id = ("NLIEx_U_mismatch", "model-10265") e.nli_visualization(nli_setting, e_config, data_loader, load_id, explain_tag)
def train_snli_ex(): hp = hyperparams.HPBert() hp.compare_deletion_num = 20 e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "SNLIEx_B" e_config.ex_val = False e_config.num_epoch = 1 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict'] #explain_tag = 'match' # 'dontcare' 'match' 'mismatch' #explain_tag = 'mismatch' #explain_tag = 'conflict' data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename, True) #load_id = ("NLI_run_nli_warm", "model-97332") #load_id = ("NLIEx_A", "model-16910") #load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') #load_id = ("NLIEx_D", "model-1964") #load_id = ("NLIEx_D", "model-1317") load_id = ("SNLI_Only_A", 'model-0') e.train_nli_any_way(nli_setting, e_config, data_loader, load_id)
def get_nli_ex_model_segmented(input_ids, input_mask, segment_ids): method = 5 hp = hyperparams.HPBert() voca_size = 30522 sequence_shape = bert_common.get_shape_list2(input_ids) batch_size = sequence_shape[0] step = 200 pad_len = 200 - 1 - (512 - (step * 2 - 1)) def spread(t): cls_token = t[:, :1] pad = tf.ones([batch_size, pad_len], tf.dtypes.int32) * PAD_ID a = t[:, :step] b = tf.concat([cls_token, t[:, step:step * 2 - 1]], axis=1) c = tf.concat([cls_token, t[:, step * 2 - 1:], pad], axis=1) return tf.concat([a, b, c], axis=0) def collect(t): a = t[:batch_size] b = t[batch_size:batch_size * 2, 1:] c = t[batch_size * 2:, 1:-pad_len] return tf.concat([a, b, c], axis=1) model = transformer_nli(hp, spread(input_ids), spread(input_mask), spread(segment_ids), voca_size, method, False) output = model.conf_logits output = collect(output) return output
def ukp_train_test(load_id, exp_name): hp = hyperparams.HPBert() e_config = ExperimentConfig() e_config.num_epoch = 2 e_config.save_interval = 100 * 60 # 30 minutes e_config.voca_size = 30522 e_config.load_names = ['bert'] encode_opt = "is_good" print(load_id) f1_list = [] for topic in data_generator.argmining.ukp_header.all_topics: e = Experiment(hp) print(exp_name) e_config.name = "arg_{}_{}_{}".format(exp_name, topic, encode_opt) data_loader = BertDataLoader(topic, True, hp.seq_max, "bert_voca.txt", option=encode_opt) save_path = e.train_ukp(e_config, data_loader, load_id) print(topic) f1_last = e.eval_ukp(e_config, data_loader, save_path) f1_list.append((topic, f1_last)) print(exp_name) print(encode_opt) print(f1_list) for key, score in f1_list: print("{0}\t{1:.03f}".format(key, score))
def predict_rf(): hp = hyperparams.HPBert() hp.batch_size = 256 e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" target_label = 'mismatch' #data_id = 'test_conflict' data_id = "{}_1000".format(target_label) e_config = ExperimentConfig() #del_g = 0.7 #e_config.name = "X_match_del_{}".format(del_g) e_config.name = "NLIEx_AnyA_{}".format(target_label) e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) #load_id = ("NLI_bare_A", 'model-195608') #load_id = ("NLIEx_O", 'model-10278') load_id = ("NLIEx_W_mismatch", "model-12030") load_id = ("NLIEx_Y_conflict", "model-12039") load_id = ("NLIEx_X_match", "model-12238") #load_id = ("NLIEx_match_del_{}".format(del_g), "model-4390") load_id = ("NLIEx_CE_{}".format(target_label), "model-12199") load_id = ("NLIEx_AnyA", "model-7255") e.predict_rf(nli_setting, e_config, data_loader, load_id, data_id, 5)
def eval_ukp_with_nli(exp_name): step_per_epoch = 24544 + 970 hp = hyperparams.HPBert() e_config = ExperimentConfig() e_config.num_steps = step_per_epoch e_config.voca_size = 30522 e_config.num_dev_batches = 30 e_config.load_names = ['bert'] encode_opt = "is_good" num_class_list = [3, 3] f1_list = [] save_path = "/mnt/scratch/youngwookim/Chair/output/model/runs/argmix_AN_B_40000_abortion_is_good/model-21306" for topic in data_generator.argmining.ukp_header.all_topics[:1]: e = Experiment(hp) print(exp_name) e_config.name = "argmix_{}_{}_{}".format(exp_name, topic, encode_opt) arg_data_loader = BertDataLoader(topic, True, hp.seq_max, "bert_voca.txt", option=encode_opt) f1_last = e.eval_ukp_on_shared(e_config, arg_data_loader, num_class_list, save_path) f1_list.append((topic, f1_last)) print(exp_name) print(encode_opt) print(f1_list) for key, score in f1_list: print("{0}\t{1:.03f}".format(key, score))
def get_nli_ex_model(input_ids, input_mask, segment_ids): method = 5 hp = hyperparams.HPBert() voca_size = 30522 model = transformer_nli(hp, input_ids, input_mask, segment_ids, voca_size, method, False) output = model.conf_logits return output
def run(token_path, ranked_list_path, start_model_path, output_path): voca_size = 30522 target_topic = get_topic_from_path(ranked_list_path) print(target_topic) hp = hyperparams.HPBert() vocab_filename = "bert_voca.txt" voca_path = os.path.join(cpath.data_path, vocab_filename) batch_size = 256 encoder = EncoderUnit(hp.seq_max, voca_path) seg_b = encoder.encoder.encode(target_topic + " is good") def encode(tokens): seg_a = encoder.encoder.ft.convert_tokens_to_ids(tokens) d = encoder.encode_inner(seg_a, seg_b) return d["input_ids"], d["input_mask"], d["segment_ids"], 0 token_reader = CluewebTokenReader(target_topic, ranked_list_path, token_path) def predict_list(batch_size, tokens_list): predictor = UkpExPredictor(hp, voca_size, start_model_path) entry_itr = [encode(tokens) for tokens in tokens_list] print("len(tokens_list)", len(tokens_list)) result = [] for idx, batch in enumerate( batch_iter_from_entry_iter(batch_size, entry_itr)): result.append(predictor.run(batch)) if idx % 100 == 0: print(idx) r = flatten_from_batches(result) print("len(r)", len(r)) return r # iterate token reader and schedule task with promise keeper pk = PromiseKeeper(partial(predict_list, batch_size)) result_list = [] for idx, doc in enumerate(token_reader.iter_docs()): future_list = [] for sent in doc: promise = MyPromise(sent, pk) future_list.append(promise.future()) result_list.append(future_list) if idx == 1000: break # encode promise into the batches and run them pk.do_duty() r = [] for future_list in result_list: r.append(list_future(future_list)) pickle.dump(r, open(output_path, "wb"))
def gradient_rte_visulize(): hp = hyperparams.HPBert() e = Experiment(hp) vocab_filename = "bert_voca.txt" load_id = loader.find_model_name("RTE_A") e_config = ExperimentConfig() e_config.name = "RTE_{}".format("visual") e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense'] data_loader = rte.DataLoader(hp.seq_max, vocab_filename, True) e.rte_visualize(e_config, data_loader, load_id)
def do_fetch_param(): hp = hyperparams.HPBert() voca_size = 30522 encode_opt = "is_good" topic = "abortion" load_run_name = "arg_nli_{}_is_good".format(topic) run_name = "arg_{}_{}_{}".format("fetch_grad", topic, encode_opt) data_loader = BertDataLoader(topic, True, hp.seq_max, "bert_voca.txt", option=encode_opt) model_path = get_model_full_path(load_run_name) names, vars = fetch_params(hp, voca_size, run_name, data_loader, model_path) r = names, vars pickle.dump(r, open(os.path.join(output_path, "params.pickle"), "wb"))
def train_nil(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLI_only_{}".format("B") e_config.num_epoch = 2 e_config.save_interval = 30 * 60 # 30 minutes data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) e.train_nli(nli_setting, e_config, data_loader)
def protest_bert(): hp = hyperparams.HPBert() e = Experiment(hp) e_config = ExperimentConfig() e_config.name = "protest" e_config.num_epoch = 1 e_config.save_interval = 1 * 60 # 1 minutes e_config.load_names = ['bert'] vocab_size = 30522 vocab_filename = "bert_voca.txt" data_loader = protest.DataLoader(hp.seq_max, vocab_filename, vocab_size) load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') e.train_protest(e_config, data_loader, load_id)
def train_rte(): hp = hyperparams.HPBert() e = Experiment(hp) vocab_filename = "bert_voca.txt" #load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') load_id = ("tlm_simple", "model.ckpt-15000") e_config = ExperimentConfig() e_config.name = "RTE_{}".format("tlm_simple_15000") e_config.num_epoch = 10 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert'] data_loader = rte.DataLoader(hp.seq_max, vocab_filename, True) e.train_rte(e_config, data_loader, load_id)
def pred_snli_ex(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "SNLIEx_B" e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("SNLIEx_B", 'model-10275') e.predict_rf(nli_setting, e_config, data_loader, load_id, "test")
def train_nil_cold(): print('train_nil_cold') hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLI_Cold" e_config.num_epoch = 2 e_config.save_interval = 30 * 60 # 30 minutes data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) saved = e.train_nli_ex_0(nli_setting, e_config, data_loader, None, False) e.test_acc2(nli_setting, e_config, data_loader, saved)
def pred_mnli_anyway(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIEx_AnyA" e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) target_label = 'mismatch' data_id = "{}_1000".format(target_label) load_id = ("NLIEx_AnyA", 'model-2785') e.predict_rf(nli_setting, e_config, data_loader, load_id, data_id)
def wikicont_bert(): hp = hyperparams.HPBert() e = Experiment(hp) e_config = ExperimentConfig() e_config.name = "WikiContrv2009_only_wiki" e_config.num_epoch = 1 e_config.save_interval = 60 * 60 # 1 minutes e_config.load_names = ['bert'] e_config.valid_freq = 100 vocab_size = 30522 vocab_filename = "bert_voca.txt" data_loader = Ams18.DataLoader(hp.seq_max, vocab_filename, vocab_size) data_loader.source_collection.collection_type = 'wiki' load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') e.train_wiki_contrv(e_config, data_loader, load_id)
def train_nli_with_premade(explain_tag): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIEx_{}".format("Premade_"+explain_tag) e_config.num_epoch = 1 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert'] #, 'cls_dense'] #, 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') e.train_nli_ex_with_premade_data(nli_setting, e_config, data_loader, load_id, explain_tag)
def test_nli(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLI_400k_tlm_simple_wo_hint" e_config.num_epoch = 2 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense'] # , 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) #saved = "/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_Cold/model-0" saved = "/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_400k_tlm_wo_hint/model-0" saved = '/mnt/scratch/youngwookim/Chair/output/model/runs/NLI_400k_tlm_simple_hint/model-0' print(saved) e.test_acc2(nli_setting, e_config, data_loader, saved)
def attribution_explain(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLI_run_{}".format("nli_eval") e_config.num_epoch = 4 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("NLI_run_nli_warm", "model-97332") load_id = ("NLI_Only_A", 'model-0') e.nli_attribution_baselines(nli_setting, e_config, data_loader, load_id)
def train_nil_on_bert(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLI_Only_{}".format("C") e_config.num_epoch = 2 e_config.save_interval = 30 * 60 # 30 minutes data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) #load_id = None load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') #load_id = ("NLI_bert_w_explain", 'model-91531') #load_id = ("NLI_Only_A", "model-0") e.train_nli_ex_0(nli_setting, e_config, data_loader, load_id, False)
def interactive(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIInterative" e_config.num_epoch = 1 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("NLI_Only_B", 'model-0') e.nli_interactive_list(nli_setting, e_config, data_loader, load_id)
def analyze_nli_pair(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIEx_pair_analyze" e_config.num_epoch = 4 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("NLIEx_T", "model-12097") e.nli_visualization_pairing(nli_setting, e_config, data_loader, load_id, data)
def predict_lime_snli_continue(): hp = hyperparams.HPBert() hp.batch_size = 512 + 256 e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "SNLI_LIME_{}".format("eval") e_config.load_names = ['bert', 'cls_dense'] data_loader = nli.SNLIDataLoader(hp.seq_max, nli_setting.vocab_filename, True) load_id = ("SNLI_Only_A", 'model-0') e.predict_lime_snli_continue(nli_setting, e_config, data_loader, load_id, "test")
def train_agree(): hp = hyperparams.HPBert() e_config = ExperimentConfig() e_config.num_epoch = 2 e_config.save_interval = 100 * 60 # 30 minutes e_config.voca_size = 30522 e_config.load_names = ['bert'] load_id = ("uncased_L-12_H-768_A-12", 'bert_model.ckpt') exp_purpose = "(dis)agree train" e = Experiment(hp) print(exp_purpose) e_config.name = "AgreeTrain" vocab_filename = "bert_voca.txt" data_loader = agree.DataLoader(hp.seq_max, vocab_filename) save_path = e.train_agree(e_config, data_loader, load_id) print(exp_purpose)
def do_test_dev_acc(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIEx_Test" e_config.load_names = ['bert', 'cls_dense'] #, 'aux_conflict'] data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) #load_id = ("NLI_bare_A", 'model-195608') load_id = ("NLIEx_S", 'model-4417') load_id = ("NLIEx_Y_conflict", "model-9636") load_id = ("NLI_Only_C", 'model-0') e.test_acc(nli_setting, e_config, data_loader, load_id)
def interactive_visual(): hp = hyperparams.HPBert() e = Experiment(hp) nli_setting = NLI() nli_setting.vocab_size = 30522 nli_setting.vocab_filename = "bert_voca.txt" e_config = ExperimentConfig() e_config.name = "NLIInterative" e_config.num_epoch = 1 e_config.save_interval = 30 * 60 # 30 minutes e_config.load_names = ['bert', 'cls_dense', 'aux_conflict'] load_id = ("NLIEx_U_mismatch", "model-10265") load_id = ("NLIEx_Y_conflict", 'model-12039') data_loader = nli.DataLoader(hp.seq_max, nli_setting.vocab_filename, True) e.nli_interactive_visual(nli_setting, e_config, data_loader, load_id)