def trainModel21(config_path): cfg = read_config(config_path) trainSemLM21Batch(cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder'], pos=cfg['pos'],max_epochs=cfg["max_epochs"], data=cfg["data"])
def calculate_selectional_preferences_parallel(config_path, model_path, list_preds, vob, output ,num_processes = 3): cfg = read_config(config_path) mdl, fm1, fm2, fm3 = loadSemLM21 (model_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder']) list_p = list(vob.keys()) for p in list_p: if p not in fm1.input_key_map: vob.pop(p) list_p = list(vob.keys()) total = len(list_p) part = total / (num_processes - 1) params = [] for i in range(num_processes - 1): start, end = int(i * part), int((i + 1) * part) params.append((vob, output, list_p, start, end, cfg, model_path)) start, end = int((num_processes - 1) * part), total params.append((vob, output, list_p, start, end, cfg, model_path)) pool = Pool() pool.map(solve, params) #for p in list_p: # print (p) # scores = get_probability_is_argument(mdl, fm, p) # process_probability(fm, scores, p, output + "/" + p + ".out.txt", vobs=vob[p]) pool.close()
def loadModel11(config_path, pred_file, output): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/best_simple.pkl" mdl, fm = loadSemLM11(load_path, cfg['train'], cfg['valid'], cfg['dep'], cfg['hidden_size'], cfg['batch_size'], cfg['save_folder'], continue_train=False) get_verb_embeddings(mdl, fm, output, fn=pred_file, embedding_layer=2)
def loadModel11(config_path, pred_file, output, parrallel=False): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/best_simple.pkl" vob = read_iSRL_candidates_new(pred_file) from models.Model11 import calculate_selectional_preferences_parallel, calculate_selectional_preferences if parrallel: calculate_selectional_preferences_parallel(config_path, load_path, list(vob.keys()), vob, output) else: calculate_selectional_preferences(config_path, load_path, list(vob.keys()), vob, output)
def loadModel22(config_path, pred_file, output): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/hybrid.pkl" mdl, fm1, fm2 = loadSemLM22(load_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'], cfg['batch_size'], cfg['save_folder'], load_dt=cfg['load_data'], continue_train=False)
def loadModel22(config_path): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/hybrid.pkl" loadSemLM22(load_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'], cfg['batch_size'], cfg['save_folder'], load_dt=cfg['load_data'], continue_train=True)
def loadModel11(config_path): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/simple.pkl.4" load_best = cfg['save_folder'] + "/best_simple.pkl" loadSemLM11(load_path, cfg['train'], cfg['valid'], cfg['dep'], cfg['hidden_size'], cfg['batch_size'], cfg['save_folder'], continue_train=True, load_best=load_best, continue_epoch=5)
def calculate_selectional_preferences(config_path, model_path, list_preds, vob, output): cfg = read_config(config_path) mdl, fm = loadSemLM11(model_path, cfg['train'], cfg['valid'], cfg['dep'], cfg['hidden_size'], cfg['batch_size'], cfg['save_folder']) for p in list_preds: scores = get_probability_is_argument(mdl, fm, p) process_probability(fm, scores, p, output + "/" + p + ".out.txt", vobs=vob[p])
def loadModel21(config_path): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/adv.pkl.29" load_best = cfg['save_folder'] + "/best_adv.pkl" loadSemLM21(load_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'], cfg['batch_size'], cfg['save_folder'], continue_train=True, load_best=load_best, continue_epoch=29)
def calculate_selectional_preferences(config_path, model_path, list_preds, vob, output ): ''' :param config_path: :param model_path: :param list_preds: :param vob: dictionary with key of preds, values are list of words :param output: :return: ''' cfg = read_config(config_path) mdl, fm1, fm2, fm3 = loadSemLM21Batch (model_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder'], pos=cfg['pos'], data=cfg['data'], load_dict=False) list_p = list(vob.keys()) for p in list_p: if p not in fm1.input_key_map: vob.pop(p) for p in vob.keys(): scores = get_probability_is_argument(mdl, fm1, fm2, fm3,p) process_probability(fm1, fm2, fm3, scores, p, output + "/" + p + ".out.txt", vobs=vob[p])
def loadModel21(config_path, pred_file, output): cfg = read_config(config_path) load_path = cfg['save_folder'] + "/best_adv.pkl" mdl, map_x1, map_x2, map_y1 = loadSemLM21(load_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'], cfg['hidden_size1'], cfg['hidden_size2'], cfg['batch_size'], cfg['save_folder'], continue_train=False) #mdl, fm1,fm2 = None,None,None get_verb_embeddings(mdl, map_x1, map_x2, map_y1, output, fn=pred_file, embedding_layer=4)
def trainModel22(config_path): cfg = read_config(config_path)
def trainModel11(config_path): cfg = read_config(config_path) trainSemLM11(cfg['train'], cfg['valid'], cfg['dep'], cfg['hidden_size'], cfg['batch_size'], cfg['save_folder'], pos=cfg['pos'], max_epochs=cfg['max_epochs'], data=cfg["data"])