batch_frac = 0.03 AL_iters = 10 for rep in range(1, num_reps): output_dir = '../data/bayesian_sequence_combination/output/ner_al_super_new/' if not os.path.isdir(output_dir): os.mkdir(output_dir) exp = Experiment(None, 9, annos.shape[1], None, max_iter=20, crf_probs=True, rep=rep) exp.save_results = True exp.opt_hyper = False#True exp.nu0_factor = 0.1 exp.alpha0_diags = 1 # best_diags exp.alpha0_factor = 1#9 # best_factor exp.methods = [ 'bac_seq_integrateIF', 'HMM_crowd', ] results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods( annos, gt, doc_start, output_dir, text, ground_truth_val=gt_val, doc_start_val=doc_start_val, text_val=text_val, ground_truth_nocrowd=gt_nocrowd, doc_start_nocrowd=doc_start_nocrowd, text_nocrowd=text_nocrowd, active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters ) # exp = Experiment(None, 9, annos.shape[1], None, max_iter=10, crf_probs=True, rep=rep) # exp.save_results = True
# ------------------------------------------------------------------------------------------------ exp = Experiment(None, 9, annos.shape[1], None, max_iter=20) exp.save_results = True exp.opt_hyper = False #True best_bac_wm = 'bac_seq' #'unknown' # choose model with best score for the different BAC worker models best_bac_wm_score = -np.inf best_nu0factor = 0.1 best_diags = 1 best_factor = 1 best_acc_bias = 0 exp.alpha0_diags = best_diags exp.alpha0_factor = best_factor exp.nu0_factor = best_nu0factor exp.alpha0_acc_bias = best_acc_bias exp.methods = [ 'majority', # 'mace', # 'ds', # 'ibcc', # 'bac_vec_integrateIF', # 'bac_ibcc_integrateIF', # 'bac_mace_integrateIF', # 'HMM_crowd', # 'best', # 'worst', # best_bac_wm,
#gt = IOB2_to_IOB(gt) # debug with subset ------- # s = 1000 # gt = gt[:s] # annos = annos[:s] # doc_start = doc_start[:s] # text = text[:s] # gt_dev = gt_dev[:s] # doc_start_dev = doc_start_dev[:s] # text_dev = text_dev[:s] # ------------------------- exp = Experiment(None, 3, annos.shape[1], None) exp.alpha0_factor = 1 exp.alpha0_diags = 100 exp.save_results = True exp.opt_hyper = False # True # run all the methods that don't require tuning here exp.methods = [ 'ibcc', 'majority', 'best', 'worst', ] seeds = [10] #np.arange(100)