inside_labels = [0] outside_labels = [1] begin_labels = [2] L = 3 confmats = {} for w, worker_model in enumerate(worker_models): outputfile = output_dir + '/EPi_list_%s.pkl' % worker_model if os.path.exists(outputfile): with open(outputfile, 'rb') as fh: EPi_list = pickle.load(fh) else: gt, annos, doc_start, text, gt_task1_dev, gt_dev, doc_start_dev, text_dev = load_data.load_biomedical_data( False, s) # # get all the gold-labelled data together goldidxs_dev = gt_task1_dev != -1 gt[goldidxs_dev] = gt_task1_dev[goldidxs_dev] nu0_factor = 0.1 alpha0_diags = 0.1 alpha0_factor = 0.1 # matrices are repeated for the different annotators/previous label conditions inside the BAC code itself. if worker_model == 'seq' or worker_model == 'ibcc' or worker_model == 'vec': f = alpha0_factor / ((L - 1) / 2) d = alpha0_diags + alpha0_factor - alpha0_factor
''' Created on April 27, 2018 @author: Edwin Simpson ''' import os import evaluation.experiment from evaluation.experiment import Experiment import data.load_data as load_data regen_data = False gt, annos, doc_start, features, gt_val, _, _, _ = load_data.load_biomedical_data( regen_data) # , debug_subset_size=900) beta0_factor = 1 alpha0_diags = 10 alpha0_factor = 10 best_begin_factor = 10 output_dir = os.path.join( evaluation.experiment.output_root_dir, 'pico3_%f_%f_%f' % (beta0_factor, alpha0_diags, alpha0_factor)) exp = Experiment(output_dir, 3, annos, gt, doc_start, features, annos, gt_val, doc_start,
if __name__ == '__main__': parser = ArgumentParser() parser.add_argument("dir", help="Directory containing the pred_xxx.csv files for recomputing performance.") parser.add_argument("dataset", help="use NER or PICO to select on of the two loaders for the gold labels") parser.add_argument("strict", help="STRICT=use strict span-level precision, recall and f1, or RELAXED=count fractions of span matches") args = parser.parse_args() if args.dataset == 'NER': gt, annos, doc_start, _, gt_nocrowd, doc_start_nocrowd, _, _, gt_val, doc_start_val, _ = \ load_data.load_ner_data(False) elif args.dataset == 'PICO': # task 1 gt, annos, doc_start, _, _, gt_nocrowd, doc_start_nocrowd, _ = load_data.load_biomedical_data(False) elif args.dataset == 'PICO2': # task 2 gt, annos, doc_start, _, _, _, _, _ = load_data.load_biomedical_data(False) gold_labelled = gt.flatten() != -1 gt_nocrowd = gt[gold_labelled] doc_start_nocrowd = doc_start[gold_labelled] else: print('Invalid dataset %s' % args.dataset) nclasses = np.max(gt) + 1 if not os.path.isdir(args.dir): print('The dir argument must specify a directory.')
''' Created on April 27, 2018 @author: Edwin Simpson ''' from evaluation.experiment import Experiment import data.load_data as load_data import numpy as np output_dir = '../../data/bayesian_sequence_combination/output/pico2/' regen_data = False gt, annos, doc_start, text, gt_task1_dev, gt_dev, doc_start_dev, text_dev = \ load_data.load_biomedical_data(regen_data) exp = Experiment(None, 3, annos.shape[1], None, max_iter=20) exp.save_results = True exp.opt_hyper = False #True # this is the one we used in the paper with O | O multiplier of 5. best_nu0factor = 1 best_diags = 10 best_factor = 100 # ------------------------------------------------------------------------------------------------ exp = Experiment(None, 3, annos.shape[1], None, max_iter=20) exp.save_results = True
''' Created on April 27, 2018 @author: Edwin Simpson ''' import os import evaluation.experiment from evaluation.experiment import Experiment import data.load_data as load_data import numpy as np regen_data = False gt, annos, doc_start, features, gt_val, _, _, _ = load_data.load_biomedical_data( regen_data) # , debug_subset_size=1000) # include this argument to debug with small dataset # ------------------------------------------------------------------------------------------------ # only hmm_Crowd actually uses these hyperparameters beta0_factor = 0.1 alpha0_diags = 0.1 alpha0_factor = 0.1 output_dir = os.path.join(evaluation.experiment.output_root_dir, 'pico3') exp = Experiment(output_dir, 3, annos, gt, doc_start, features, annos,