def trainModel21(config_path):
cfg = read_config(config_path)
trainSemLM21Batch(cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'],
cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder'],
pos=cfg['pos'],max_epochs=cfg["max_epochs"], data=cfg["data"])
def calculate_selectional_preferences_parallel(config_path, model_path, list_preds, vob, output ,num_processes = 3):
cfg = read_config(config_path)
mdl, fm1, fm2, fm3 = loadSemLM21 (model_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'],
cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder'])
list_p = list(vob.keys())
for p in list_p:
if p not in fm1.input_key_map:
vob.pop(p)
list_p = list(vob.keys())
total = len(list_p)
part = total / (num_processes - 1)
params = []
for i in range(num_processes - 1):
start, end = int(i * part), int((i + 1) * part)
params.append((vob, output, list_p, start, end, cfg, model_path))
start, end = int((num_processes - 1) * part), total
params.append((vob, output, list_p, start, end, cfg, model_path))
pool = Pool()
pool.map(solve, params)
#for p in list_p:
# print (p)
# scores = get_probability_is_argument(mdl, fm, p)
# process_probability(fm, scores, p, output + "/" + p + ".out.txt", vobs=vob[p])
pool.close()
def loadModel11(config_path, pred_file, output):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/best_simple.pkl"
mdl, fm = loadSemLM11(load_path,
cfg['train'],
cfg['valid'],
cfg['dep'],
cfg['hidden_size'],
cfg['batch_size'],
cfg['save_folder'],
continue_train=False)
get_verb_embeddings(mdl, fm, output, fn=pred_file, embedding_layer=2)
def loadModel11(config_path, pred_file, output, parrallel=False):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/best_simple.pkl"
vob = read_iSRL_candidates_new(pred_file)
from models.Model11 import calculate_selectional_preferences_parallel, calculate_selectional_preferences
if parrallel:
calculate_selectional_preferences_parallel(config_path, load_path,
list(vob.keys()), vob,
output)
else:
calculate_selectional_preferences(config_path, load_path,
list(vob.keys()), vob, output)
def loadModel22(config_path, pred_file, output):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/hybrid.pkl"
mdl, fm1, fm2 = loadSemLM22(load_path,
cfg['train'],
cfg['valid'],
cfg['we_dict1'],
cfg['we_dict2'],
cfg['dep'],
cfg['hidden_size1'],
cfg['hidden_size2'],
cfg['batch_size'],
cfg['save_folder'],
load_dt=cfg['load_data'],
continue_train=False)
def loadModel22(config_path):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/hybrid.pkl"
loadSemLM22(load_path,
cfg['train'],
cfg['valid'],
cfg['we_dict1'],
cfg['we_dict2'],
cfg['dep'],
cfg['hidden_size1'],
cfg['hidden_size2'],
cfg['batch_size'],
cfg['save_folder'],
load_dt=cfg['load_data'],
continue_train=True)
def loadModel11(config_path):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/simple.pkl.4"
load_best = cfg['save_folder'] + "/best_simple.pkl"
loadSemLM11(load_path,
cfg['train'],
cfg['valid'],
cfg['dep'],
cfg['hidden_size'],
cfg['batch_size'],
cfg['save_folder'],
continue_train=True,
load_best=load_best,
continue_epoch=5)
def calculate_selectional_preferences(config_path, model_path, list_preds, vob,
output):
cfg = read_config(config_path)
mdl, fm = loadSemLM11(model_path, cfg['train'], cfg['valid'], cfg['dep'],
cfg['hidden_size'], cfg['batch_size'],
cfg['save_folder'])
for p in list_preds:
scores = get_probability_is_argument(mdl, fm, p)
process_probability(fm,
scores,
p,
output + "/" + p + ".out.txt",
vobs=vob[p])
def loadModel21(config_path):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/adv.pkl.29"
load_best = cfg['save_folder'] + "/best_adv.pkl"
loadSemLM21(load_path,
cfg['train'],
cfg['valid'],
cfg['we_dict1'],
cfg['we_dict2'],
cfg['dep'],
cfg['hidden_size1'],
cfg['hidden_size2'],
cfg['batch_size'],
cfg['save_folder'],
continue_train=True,
load_best=load_best,
continue_epoch=29)
def calculate_selectional_preferences(config_path, model_path, list_preds, vob, output ):
'''
:param config_path:
:param model_path:
:param list_preds:
:param vob: dictionary with key of preds, values are list of words
:param output:
:return:
'''
cfg = read_config(config_path)
mdl, fm1, fm2, fm3 = loadSemLM21Batch (model_path, cfg['train'], cfg['valid'], cfg['we_dict1'], cfg['we_dict2'], cfg['dep'],
cfg['hidden_size1'], cfg['hidden_size2'],cfg['batch_size'], cfg['save_folder'], pos=cfg['pos'], data=cfg['data'], load_dict=False)
list_p = list(vob.keys())
for p in list_p:
if p not in fm1.input_key_map:
vob.pop(p)
for p in vob.keys():
scores = get_probability_is_argument(mdl, fm1, fm2, fm3,p)
process_probability(fm1, fm2, fm3, scores, p, output + "/" + p + ".out.txt", vobs=vob[p])
def loadModel21(config_path, pred_file, output):
cfg = read_config(config_path)
load_path = cfg['save_folder'] + "/best_adv.pkl"
mdl, map_x1, map_x2, map_y1 = loadSemLM21(load_path,
cfg['train'],
cfg['valid'],
cfg['we_dict1'],
cfg['we_dict2'],
cfg['dep'],
cfg['hidden_size1'],
cfg['hidden_size2'],
cfg['batch_size'],
cfg['save_folder'],
continue_train=False)
#mdl, fm1,fm2 = None,None,None
get_verb_embeddings(mdl,
map_x1,
map_x2,
map_y1,
output,
fn=pred_file,
embedding_layer=4)
def trainModel22(config_path):
cfg = read_config(config_path)
def trainModel11(config_path):
cfg = read_config(config_path)
trainSemLM11(cfg['train'], cfg['valid'], cfg['dep'],
cfg['hidden_size'], cfg['batch_size'], cfg['save_folder'], pos=cfg['pos'], max_epochs=cfg['max_epochs'], data=cfg["data"])
