def train_svm(train_processed_path): # Load abbr index abbr_idx_mapper = pickle_reader(train_processed_path + '/abbr_idx_mapper.pkl') abbr_cui2idx_inventory = {} os.makedirs(train_processed_path + '/svm_models/', exist_ok=True) # generate training data & train model for abbr, abbr_idx in tqdm.tqdm(abbr_idx_mapper['abbr2idx'].items()): content_vector = pickle_reader(train_processed_path + '/content_vectors/%d_vector.pkl' % abbr_idx) label2idx = {} label_idx = 0 x = [] y = [] for global_instance_idx, doc_id, pos, content_pos, content_vec, content, label in content_vector: if label not in label2idx: label2idx[label] = label_idx label_idx += 1 x.append(content_vec) y.append(label2idx[label]) abbr_cui2idx_inventory[abbr] = label2idx # no need to train if only have 1 CUI if len(label2idx) > 1: x_train, y_train = train_sample(x, y, 2000) # train svm model model = SVC(kernel='rbf', gamma=0.01, C=100).fit(x_train, y_train) pickle_writer( model, train_processed_path + '/svm_models/%d_svm.pkl' % abbr_idx) pickle_writer(abbr_cui2idx_inventory, train_processed_path + '/abbr_cui_idx_inventory.pkl')
def evaluate_score_svm(test_processed_path, train_processed_path): # Load abbr index abbr_idx_mapper = pickle_reader(test_processed_path + '/abbr_idx_mapper.pkl') abbr_idx_mapper_train = pickle_reader(train_processed_path + '/abbr_idx_mapper.pkl') abbr2train_idx = abbr_idx_mapper_train['abbr2idx'] abbr_cui2idx_inventory = pickle_reader(train_processed_path + '/abbr_cui_idx_inventory.pkl') count_correct = 0 count_all = 0 count_model_correct = 0 count_model_all = 0 count_no_label = 0 count_correct_without_predict = 0 # generate testing data for abbr, abbr_idx in tqdm.tqdm(abbr_idx_mapper['abbr2idx'].items()): content_vector = pickle_reader(test_processed_path + '/content_vectors/%d_vector.pkl' % abbr_idx) if abbr not in abbr_cui2idx_inventory: count_all += len(content_vector) count_no_label += len(content_vector) else: label2idx = abbr_cui2idx_inventory[abbr] count_all += len(content_vector) x = [] y = [] for _, _, _, _, content_vec, _, label in content_vector: # if true label not in train collection if label not in label2idx: count_no_label += 1 # if only have 1 CUI elif len(label2idx) == 1: count_correct += 1 count_correct_without_predict += 1 # need predict else: x.append(content_vec) y.append(label2idx[label]) # predict if len(y) > 0: count_model_all += len(y) model = pickle_reader(train_processed_path + '/svm_models/%d_svm.pkl' % abbr2train_idx[abbr]) y_pred = model.predict(np.vstack(x)) temp_correct = sum(y == y_pred) count_correct += temp_correct count_model_correct += temp_correct print("DataSet Accuracy (all instances): ", count_correct / count_all) print("Model Accuracy (only ambiguous instances): ", count_model_correct / count_model_all) print("Num.instances: ", count_all) print("Num.gt abbr-CUI mapping not found: ", count_no_label) print("Num.correct without predict", count_correct_without_predict) print()
def predict_fasttext_classifier(train_processed_path, test_processed_path, use_pretrain=False, use_softmax=False): train_path = train_processed_path + '/fasttext' if use_pretrain: model_path = train_path + '/model/pre_train' else: model_path = train_path + '/model' # Load abbr index train_abbr_idx_mapper = pickle_reader(train_path + '/abbr_idx_mapper.pkl') train_abbr2idx = train_abbr_idx_mapper['abbr2idx'] test_abbr_idx_mapper = pickle_reader(test_processed_path + '/fasttext/abbr_idx_mapper.pkl') test_abbr_index = AbbrIndex(test_processed_path + '/abbr_index_data.pkl') # Load model if use_softmax: model_file = model_path + '/all_softmax.bin' else: model_file = model_path + '/all.bin' model = load_model(model_file) label_set = set(map(lambda x: x.lstrip("__label__"), model.get_labels())) instance_collection = [] # generate testing data for abbr, test_abbr_idx in tqdm(test_abbr_idx_mapper['abbr2idx'].items()): # if abbr not in train_abbr2idx: # for doc_id, pos_list in test_abbr_index[abbr].items(): # for global_instance_idx, pos, label in pos_list: # instance_collection.append(InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) # else: eval_abbr_instance_list = txt_reader(test_processed_path + '/fasttext/dataset/%d.txt' % test_abbr_idx) abbr_instance_idx = 0 for doc_id, pos_list in test_abbr_index[abbr].items(): for global_instance_idx, pos, label in pos_list: if label not in label_set: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) else: # get instance tokens = eval_abbr_instance_list[abbr_instance_idx].split() label_in_txt = tokens[0].lstrip("__label__") assert label == label_in_txt context = " ".join(tokens[1:]) instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=model.predict(context)[0] [0].lstrip("__label__"))) abbr_instance_idx += 1 # sort collection list based on global instance idx instance_collection = sorted(instance_collection, key=lambda x: x.index) return instance_collection
def svm_cross_validation(train_processed_path, abbr_idx=0): """ Tuning the parameters on largest abbr in train set. :param train_processed_path: :param abbr_idx: :return: """ content_vector = pickle_reader(train_processed_path + '/content_vectors/%d_vector.pkl' % abbr_idx) label2idx = {} label_idx = 0 x = [] y = [] for instance_id, doc_id, pos, content_pos, content_vec, content, label in content_vector: if label not in label2idx: label2idx[label] = label_idx label_idx += 1 x.append(content_vec) y.append(label2idx[label]) x_train, y_train = train_sample(x, y, 500) parameters = {'gamma': [1e-4, 1e-3, 1e-2], 'C': [1e-1, 1, 10, 100, 1000]} model = SVC(kernel='rbf') model_cv = GridSearchCV(model, parameters, cv=5).fit(x_train, y_train) print(model_cv.best_params_) print(model_cv.best_score_) return model_cv
def generate_whole_dataset(processed_path, shuffle=False): abbr_idx_mapper = pickle_reader(processed_path + '/fasttext/abbr_idx_mapper.pkl') with open(processed_path + '/fasttext/dataset/all.txt', 'w') as f: total_dataset = [] for abbr, abbr_idx in tqdm(abbr_idx_mapper['abbr2idx'].items()): total_dataset.extend( txt_reader(processed_path + '/fasttext/dataset/%d.txt' % abbr_idx)) if shuffle: random.shuffle(total_dataset) f.write("\n".join(total_dataset))
def train_fasttext_classifier_multi_model(train_processed_path, use_pretrain=False, use_softmax=False): train_path = train_processed_path + '/fasttext' if use_pretrain: model_path = train_path + '/model/pre_train' else: model_path = train_path + '/model' if use_softmax: loss = 'softmax' else: loss = 'hs' os.makedirs(model_path, exist_ok=True) # Load abbr index abbr_idx_mapper = pickle_reader(train_path + '/abbr_idx_mapper.pkl') abbr_index = AbbrIndex(train_processed_path + '/abbr_index_data.pkl') abbr_label_set = {} # Load training data & train model for abbr, abbr_idx in tqdm(abbr_idx_mapper['abbr2idx'].items()): input_file = train_path + '/dataset/%d.txt' % abbr_idx model_file = model_path + '/%d.bin' % abbr_idx # load label list label_set = set() for doc_id, pos_list in abbr_index[abbr].items(): for global_instance_idx, pos, label in pos_list: label_set.add(label) abbr_label_set[abbr] = label_set # no need to train if only have 1 CUI if len(label_set) > 1: model_config = { 'input': input_file, 'epoch': 50, 'lr': 0.1, 'lrUpdateRate': 100, 'dim': 100, 'ws': 5, 'wordNgrams': 2, 'loss': loss, 'thread': 60, } if use_pretrain: model_config[ 'pretrainedVectors'] = train_processed_path + '/fasttext.vec' model = train_supervised(**model_config) model.save_model(model_file) pickle_writer(abbr_label_set, train_path + '/abbr_label_set.pkl')
def train_fasttext_classifier(train_processed_path, abbr=None, use_pretrain=False, use_softmax=False): train_path = train_processed_path + '/fasttext' if use_pretrain: model_path = train_path + '/model/pre_train' else: model_path = train_path + '/model' train_path = train_processed_path + '/fasttext' os.makedirs(model_path, exist_ok=True) if abbr is None: # train on whole dataset input_file = train_path + '/dataset/all.txt' if use_softmax: model_file = model_path + '/all_softmax.bin' else: model_file = model_path + '/all.bin' else: # Load abbr index abbr_idx_mapper = pickle_reader(train_path + '/abbr_idx_mapper.pkl') abbr_idx = abbr_idx_mapper['abbr2idx'][abbr] input_file = train_path + '/dataset/%d.txt' % abbr_idx model_file = model_path + '/%d.bin' % abbr_idx if use_softmax: loss = 'softmax' else: loss = 'hs' model_config = { 'input': input_file, 'epoch': 50, 'lr': 0.1, 'lrUpdateRate': 100, 'dim': 100, 'ws': 5, 'wordNgrams': 2, 'loss': loss, 'thread': 60, } if use_pretrain: model_config[ 'pretrainedVectors'] = train_processed_path + '/fasttext.vec' model = train_supervised(**model_config) model.save_model(model_file) return model
txt_list_processed_sorted = sorted(txt_list_processed, key=operator.itemgetter(0)) return [txt for _, txt in txt_list_processed_sorted] if __name__ == '__main__': ###################################### # Read texts from dataset ###################################### # BASE_FOLDER = '/home/mengr/Project/wsd/wsd_data/' dataset_paths = DataSetPaths(environment='luoz3_x1') DATASET_PATH = dataset_paths.upmc_all_no_mark_txt OUTPUT_PATH = dataset_paths.upmc_all_no_mark_folder PATH_PROCESSED_INVENTORY_PKL = dataset_paths.sense_inventory_pkl # Get pickle generated from mimic_inventory.py inventory = pickle_reader(PATH_PROCESSED_INVENTORY_PKL) inventory_rmapper = inventory['longform-abbr_cui'] ###################################### # Processing ###################################### txt_list = list(open(DATASET_PATH, 'r').readlines()) print("Loaded %d docs from %s" % (len(txt_list), DATASET_PATH)) # Replace Long forms to abbrs mimic_txt_processed = longform_replacer(txt_list, inventory_rmapper, n_jobs=50) # Save to file txt_writer(mimic_txt_processed, OUTPUT_PATH+'train_no_mark_longform_replaced.txt')
def predict_fasttext_classifier_multi_model(train_processed_path, test_processed_path, use_pretrain=False): train_path = train_processed_path + '/fasttext' if use_pretrain: model_path = train_path + '/model/pre_train' else: model_path = train_path + '/model' # Load abbr index test_abbr_idx_mapper = pickle_reader(test_processed_path + '/fasttext/abbr_idx_mapper.pkl') test_abbr_index = AbbrIndex(test_processed_path + '/abbr_index_data.pkl') train_abbr_idx_mapper = pickle_reader(train_processed_path + '/fasttext/abbr_idx_mapper.pkl') train_abbr2idx = train_abbr_idx_mapper['abbr2idx'] train_abbr_label_set = pickle_reader(train_processed_path + '/fasttext/abbr_label_set.pkl') instance_collection = [] # generate testing data for abbr, test_abbr_idx in tqdm(test_abbr_idx_mapper['abbr2idx'].items()): if abbr not in train_abbr_label_set: for doc_id, pos_list in test_abbr_index[abbr].items(): for global_instance_idx, pos, label in pos_list: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) else: train_label_set = train_abbr_label_set[abbr] eval_abbr_instance_list = txt_reader(test_processed_path + '/fasttext/dataset/%d.txt' % test_abbr_idx) abbr_instance_idx = 0 context_list, global_idx_list = [], [] for doc_id, pos_list in test_abbr_index[abbr].items(): for global_instance_idx, pos, label in pos_list: # if true label not in train collection if label not in train_label_set: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) # if only have 1 CUI elif len(train_label_set) == 1: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=label)) # need predict else: # get instance tokens = eval_abbr_instance_list[ abbr_instance_idx].split() label_in_txt = tokens[0].lstrip("__label__") assert label == label_in_txt context = " ".join(tokens[1:]) context_list.append(context) global_idx_list.append(global_instance_idx) abbr_instance_idx += 1 # predict if len(context_list) > 0: # Load model model_file = model_path + '/%d.bin' % train_abbr2idx[abbr] model = load_model(model_file) predict_list = model.predict(context_list)[0] for idx, predict in zip(global_idx_list, predict_list): instance_collection.append( InstancePred( index=idx, abbr=abbr, sense_pred=predict[0].lstrip("__label__"))) # sort collection list based on global instance idx instance_collection = sorted(instance_collection, key=lambda x: x.index) return instance_collection
def predict_svm(test_processed_path, train_processed_path): # Load abbr index abbr_idx_mapper = pickle_reader(test_processed_path + '/abbr_idx_mapper.pkl') abbr_idx_mapper_train = pickle_reader(train_processed_path + '/abbr_idx_mapper.pkl') abbr2train_idx = abbr_idx_mapper_train['abbr2idx'] abbr_cui2idx_inventory = pickle_reader(train_processed_path + '/abbr_cui_idx_inventory.pkl') instance_collection = [] # generate testing data for abbr, abbr_idx in tqdm.tqdm(abbr_idx_mapper['abbr2idx'].items()): content_vector = pickle_reader(test_processed_path + '/content_vectors/%d_vector.pkl' % abbr_idx) if abbr not in abbr_cui2idx_inventory: for global_instance_idx, _, _, _, _, _, _ in content_vector: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) else: label2idx = abbr_cui2idx_inventory[abbr] x = [] y = [] global_idx_list = [] for global_instance_idx, _, _, _, content_vec, _, label in content_vector: # if true label not in train collection if label not in label2idx: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=None)) # if only have 1 CUI elif len(label2idx) == 1: instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=label)) # need predict else: x.append(content_vec) y.append(label2idx[label]) global_idx_list.append(global_instance_idx) # predict if len(y) > 0: model = pickle_reader(train_processed_path + '/svm_models/%d_svm.pkl' % abbr2train_idx[abbr]) y_pred = model.predict(np.vstack(x)) # get idx2label idx2label = {} for label, idx in label2idx.items(): idx2label[idx] = label for idx_pred, global_instance_idx in zip( y_pred, global_idx_list): instance_collection.append( InstancePred(index=global_instance_idx, abbr=abbr, sense_pred=idx2label[idx_pred])) # sort collection list based on global instance idx instance_collection = sorted(instance_collection, key=lambda x: x.index) return instance_collection