num_reps = 10
batch_frac = 0.03
AL_iters = 10
for rep in range(1, num_reps):
output_dir = '../data/bayesian_sequence_combination/output/ner_al_super_new/'
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
exp = Experiment(None, 9, annos.shape[1], None, max_iter=20, crf_probs=True, rep=rep)
exp.save_results = True
exp.opt_hyper = False#True
exp.nu0_factor = 0.1
exp.alpha0_diags = 1 # best_diags
exp.alpha0_factor = 1#9 # best_factor
exp.methods = [
'bac_seq_integrateIF',
'HMM_crowd',
]
results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods(
annos, gt, doc_start, output_dir, text,
ground_truth_val=gt_val, doc_start_val=doc_start_val, text_val=text_val,
ground_truth_nocrowd=gt_nocrowd, doc_start_nocrowd=doc_start_nocrowd, text_nocrowd=text_nocrowd,
active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters
)
# exp = Experiment(None, 9, annos.shape[1], None, max_iter=10, crf_probs=True, rep=rep)
load_data.load_ner_data(regen_data)
# ------------------------------------------------------------------------------------------------
exp = Experiment(None, 9, annos.shape[1], None, max_iter=20)
exp.save_results = True
exp.opt_hyper = False #True
best_bac_wm = 'bac_seq' #'unknown' # choose model with best score for the different BAC worker models
best_bac_wm_score = -np.inf
best_nu0factor = 0.1
best_diags = 1
best_factor = 1
best_acc_bias = 0
exp.alpha0_diags = best_diags
exp.alpha0_factor = best_factor
exp.nu0_factor = best_nu0factor
exp.alpha0_acc_bias = best_acc_bias
exp.methods = [
'majority',
# 'mace',
# 'ds',
# 'ibcc',
# 'bac_vec_integrateIF',
# 'bac_ibcc_integrateIF',
# 'bac_mace_integrateIF',
# 'HMM_crowd',
# 'best',
# 'worst',
# debug with subset -------
# s = 1000
# gt = gt[:s]
# annos = annos[:s]
# doc_start = doc_start[:s]
# text = text[:s]
# gt_dev = gt_dev[:s]
# doc_start_dev = doc_start_dev[:s]
# text_dev = text_dev[:s]
# -------------------------
exp = Experiment(None, 3, annos.shape[1], None)
exp.alpha0_factor = 1
exp.alpha0_diags = 100
exp.save_results = True
exp.opt_hyper = False # True
# run all the methods that don't require tuning here
exp.methods = [
'ibcc',
'majority',
'best',
'worst',
]
seeds = [10] #np.arange(100)
annos = annos[gt.flatten() != -1]
