# ------------------------------------------------------------------------------------------------ exp = Experiment(None, 9, annos.shape[1], None, max_iter=20) exp.save_results = True exp.opt_hyper = False #True best_bac_wm = 'bac_seq' #'unknown' # choose model with best score for the different BAC worker models best_bac_wm_score = -np.inf best_nu0factor = 0.1 best_diags = 1 best_factor = 1 best_acc_bias = 0 exp.alpha0_diags = best_diags exp.alpha0_factor = best_factor exp.nu0_factor = best_nu0factor exp.alpha0_acc_bias = best_acc_bias exp.methods = [ 'majority', # 'mace', # 'ds', # 'ibcc', # 'bac_vec_integrateIF', # 'bac_ibcc_integrateIF', # 'bac_mace_integrateIF', # 'HMM_crowd', # 'best', # 'worst', # best_bac_wm, 'bac_seq_integrateIF',
num_reps = 10 batch_frac = 0.03 AL_iters = 10 for rep in range(1, num_reps): output_dir = '../data/bayesian_sequence_combination/output/ner_al_super_new/' if not os.path.isdir(output_dir): os.mkdir(output_dir) exp = Experiment(None, 9, annos.shape[1], None, max_iter=20, crf_probs=True, rep=rep) exp.save_results = True exp.opt_hyper = False#True exp.nu0_factor = 0.1 exp.alpha0_diags = 1 # best_diags exp.alpha0_factor = 1#9 # best_factor exp.methods = [ 'bac_seq_integrateIF', 'HMM_crowd', ] results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods( annos, gt, doc_start, output_dir, text, ground_truth_val=gt_val, doc_start_val=doc_start_val, text_val=text_val, ground_truth_nocrowd=gt_nocrowd, doc_start_nocrowd=doc_start_nocrowd, text_nocrowd=text_nocrowd, active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters )