def train_iteration(opts, assoc_model, trainer, do_sym, log_file):
"""
Setup where association scores are learned, relation by relation.
based on `model.macro_loops()`
:return: full-graph iteration scores
"""
ep_loss = 0.0
# iterate over relations:
graphs_order = list(assoc_model.graphs.items())
# TODO maybe even randomize edge order across relations
if opts.rand_nodes:
dy.np.random.shuffle(graphs_order)
for rel, g in graphs_order:
# report
if opts.v > 0:
timeprint('starting loop over {}'.format(rel))
if opts.rule_override and rel in SYMMETRIC_RELATIONS and not do_sym:
timeprint('RELATION OVERRIDEN')
continue
# iterate over nodes (each as source + as target):
node_order = list(range(N))
if opts.rand_nodes:
dy.np.random.shuffle(node_order)
for node in tqdm(node_order):
if opts.debug and node % 100 != 0:
continue
ep_loss += node_iteration(rel, g, node, opts, assoc_model, trainer, log_file, is_source=True)
ep_loss += node_iteration(rel, g, node, opts, assoc_model, trainer, log_file, is_source=False)
return ep_loss
def __init__(self, graphs, embeddings, mode=TRANSLATIONAL_EMBED_MODE, dropout=0.0, model_path=None):
"""
:param graphs: dictionary of <relation:CSR-format graph>s, node-aligned
:param embeddings: list of numpy array embeddings, indices aligned to nodes
:param mode: mode of calculating association score, options: {}
""".format(MODES_STR)
# input validation
graph_sizes = list(set([g.shape[0] for g in list(graphs.values())]))
assert len(graph_sizes) == 1
assert len(embeddings) == graph_sizes[0], '{} != {}'.format(len(embeddings), graph_sizes[0])
# raw members
self.graphs = {canonicalize_name(k):g for k,g in list(graphs.items())}
self.mode = mode
# documenationy members
self.relation_names = sorted(self.graphs.keys())
if 'co_hypernym' in self.relation_names:
self.relation_names.remove('co_hypernym')
self.vocab_size = graph_sizes[0]
self.R = len(self.relation_names)
self.emb_dim = len(embeddings[0])
self.dropout = dropout
# model members
self.model = dy.Model()
# TODO consider using no_update param for embeddings
self.embeddings = self.model.add_lookup_parameters((self.vocab_size, self.emb_dim))
self.embeddings.init_from_array(embeddings)
# init association parameter
self.no_assoc = False # so can be overriden in inheritors
# first determine
if self.mode == BILINEAR_MODE: # full-rank bilinear matrix
assoc_dim = (self.emb_dim, self.emb_dim)
elif self.mode == DIAG_RANK1_MODE: # diagonal bilinear matrix + rank 1 matrix
# first row = diagonal
# second row = 'source factor'
# third row = 'target factor'
assoc_dim = (3, self.emb_dim)
elif self.mode == TRANSLATIONAL_EMBED_MODE: # additive relational vector
assoc_dim = self.emb_dim
elif self.mode == DISTMULT: # diagonal bilinear matrix
assoc_dim = self.emb_dim
else:
raise ValueError('unsupported mode: {}. allowed are {}'\
.format(self.mode, ', '.join(MODES_STR)))
# init actual parameter
self.word_assoc_weights = {r:self.model.add_parameters(assoc_dim) for r in self.relation_names}
if model_path is not None:
self.model.populate(model_path + '.dyn')
timeprint('finished initialization for association model.')
def init_ergm_features(self, graphs=None):
"""
Computes ERGM features from scratch, populates cache members and self.feature_vals
:param graphs: if not None, changes underlying member and inits from it.
"""
if graphs is not None:
self.graphs = graphs
self.feature_vals = {}
self.init_ergm_cache()
self.update_features()
timeprint('initialized features from cache')
def init_ergm_features(self, graphs=None):
"""
Computes ERGM features from scratch, populates cache members and self.feature_vals
:param graphs: if not None, changes underlying member and inits from it.
"""
if graphs is not None:
self.graphs = graphs
self.feature_vals = {}
self.init_ergm_cache()
self.update_features()
timeprint('initialized features from cache')
def train_iteration(opts, assoc_model, trainer, do_sym, log_file):
"""
Setup where association scores are learned, relation by relation.
based on `model.macro_loops()`
:return: full-graph iteration scores
"""
ep_loss = 0.0
# iterate over relations:
graphs_order = list(assoc_model.graphs.items())
# TODO maybe even randomize edge order across relations
if opts.rand_nodes:
dy.np.random.shuffle(graphs_order)
for rel, g in graphs_order:
# report
if opts.v > 0:
timeprint('starting loop over {}'.format(rel))
if opts.rule_override and rel in SYMMETRIC_RELATIONS and not do_sym:
timeprint('RELATION OVERRIDEN')
continue
# iterate over nodes (each as source + as target):
node_order = list(range(N))
if opts.rand_nodes:
dy.np.random.shuffle(node_order)
for node in tqdm(node_order):
if opts.debug and node % 100 != 0:
continue
ep_loss += node_iteration(rel,
g,
node,
opts,
assoc_model,
trainer,
log_file,
is_source=True)
ep_loss += node_iteration(rel,
g,
node,
opts,
assoc_model,
trainer,
log_file,
is_source=False)
return ep_loss
def node_iteration(rel, g, node, opts, assoc_model, trainer, log_file, is_source):
"""
Perform one iteration of trying to score a node's neighbors above negative samples.
"""
# true instances likelihood
trues = targets(g, node) if is_source else sources(g, node)
side = '->' if is_source else '<-'
if len(trues) == 0: return 0.0
if opts.debug:
dy.renew_cg(immediate_compute = True, check_validity = True)
else:
dy.renew_cg()
# compute association score as dynet expression (can't do this above due to staleness)
true_scores = []
for tr in trues:
if is_source:
j_assoc_score = assoc_model.word_assoc_score(node, tr, rel)
else:
j_assoc_score = assoc_model.word_assoc_score(tr, node, rel)
if log_file is not None:
log_file.write('{} {}\tTRUE_{}\t{:.3e}\n'\
.format(node, side, tr, j_assoc_score.scalar_value()))
true_scores.append(j_assoc_score)
# false targets likelihood - negative sampling (uniform)
# collect negative samples
if opts.nll:
sample_scores = [[ts] for ts in true_scores]
else:
margins = []
neg_samples = [np.random.choice(range(N)) for _ in range(opts.neg_samp * len(trues))]
# remove source and true targets if applicable
for t in [node] + trues:
if t in neg_samples:
neg_samples.remove(t)
neg_samples.append(np.random.choice(range(N)))
for (i,ns) in enumerate(neg_samples):
# compute association score as dynet expression
if is_source:
ns_assoc_score = assoc_model.word_assoc_score(node, ns, rel)
else:
ns_assoc_score = assoc_model.word_assoc_score(ns, node, rel)
if log_file is not None:
log_file.write('{} {}\tNEG_{}\t{:.3e}\n'\
.format(node, side, ns, ns_assoc_score.scalar_value()))
corresponding_true = i // opts.neg_samp
if opts.nll:
sample_scores[corresponding_true].append(ns_assoc_score)
else:
# TODO maybe use dy.hinge()
ctt_score = true_scores[corresponding_true]
margin = ctt_score - ns_assoc_score
margins.append(dy.rectify(dy.scalarInput(1.0) - margin))
# compute overall loss
if opts.nll:
if len(sample_scores) == 0:
dy_loss = dy.scalarInput(0.0)
else:
dy_loss = dy.esum([dy.pickneglogsoftmax(dy.concatenate(scrs), 0) for scrs in sample_scores])
else:
if len(margins) == 0:
dy_loss = dy.scalarInput(0.0)
else:
dy_loss = dy.esum(margins)
sc_loss = dy_loss.scalar_value()
if log_file is not None:
log_file.write('{}\tLOSS\t{:.3e}\n'\
.format(node, sc_loss))
# backprop and recompute score
if opts.v > 1:
timeprint('overall loss for relation {}, node {} as {} = {:.6f}'\
.format(rel, node, 'source' if is_source else 'target', sc_loss))
dy_loss.backward()
trainer.update()
return sc_loss
def remove_edge(self,
src_idx,
trg_idx,
rel,
update_feats=True,
permanent=True,
caches=None,
report_feat_diff=False):
"""
Removes edge from graph, updates cache and feature values
:param src_idx: index of source node from edge to remove
:param trg_idx: index of target node from edge to remove
:param rel: relation type
:param update_feats: flag for not updating all cache and features, to be deferred
:returns: if permanent=False, returns ergm score of removed-edge graph
"""
if permanent:
self.score_is_stale = True
cached_feats = None
cached_cache = None
else:
if caches is not None:
cached_cache = copy.deepcopy(caches[0])
cached_feats = caches[1]
else:
cached_cache = copy.deepcopy(self.cache)
cached_feats = copy.deepcopy(self.feature_vals)
update_feats = True # no other mode possible
# update cache members
# decrement edge count for rel
self.edge_counts[rel] -= 1
# pair cache members
for r, g in list(self.graphs.items()):
if rel == 'hypernym' and r == 'co_hypernym':
continue
# decrement mutual edge count for pairs with trg-src edges
if g[trg_idx, src_idx] == 1:
self.mutual_edge_counts[find_key(self.mutual_edge_counts,
(rel, r))] -= 1
# decrement two-paths for x-src-trg and src-trg-y
self.two_path_counts[(r, rel)] -= self.in_degs[r][0, src_idx]
self.two_path_counts[(rel, r)] -= self.out_degs[r][0, trg_idx]
# triplet cache members
for r2, g2 in list(self.graphs.items()):
if rel == 'hypernym' and r2 == 'co_hypernym':
continue
# decrement transitive closures from two-paths src-x-trg
if self.out_degs[r][0, src_idx] > 0 and self.in_degs[r2][
0, trg_idx] > 0:
r_r2_betweens = (g[src_idx] * g2[:, trg_idx]).sum()
self.transitive_closure_counts[(r, r2,
rel)] -= r_r2_betweens
# decrement directed triangle count
if self.out_degs[r2][0, trg_idx] > 0 and self.in_degs[r][
0, src_idx] > 0:
r_r2_cycles = (g2[trg_idx] * g[:, src_idx]).sum()
rs_key = find_cyclic_key(self.directed_triangle_counts,
(r, rel, r2))
self.directed_triangle_counts[rs_key] -= r_r2_cycles
# decrement src's out_degree and trg's in_degree in rel and update all related caches
self.out_degs[rel][0, src_idx] -= 1
self.in_degs[rel][0, trg_idx] -= 1
if update_feats:
# recompute heavy cache updates from raw counts
self.update_stars_cache_from_edge(rel,
src_idx,
trg_idx,
added=False)
# update features from caches
self.update_features()
if not permanent and report_feat_diff:
timeprint('changed feature values:')
diff_keys = [
k for k in self.feature_vals
if self.feature_vals[k] != cached_feats[k]
]
if len(diff_keys) > 0:
print('\n'.join(['{}: from {} to {}'\
.format(k, cached_feats[k], self.feature_vals[k]) for k in diff_keys]))
if permanent:
# remove actual edge
self.graphs[rel][src_idx, trg_idx] = 0
else:
if rel == 'hypernym':
self.zero_all_feats('co_hypernym')
# prepare return value
ret = self.ergm_score()
# revert everything
self.reread_cache(cached_cache)
self.feature_vals = cached_feats
# return prepared score
return ret
parser.add_argument("--neg-samp", type=int, default=ASSOC_DEFAULT_NEGS, help="nodes for negative sample")
parser.add_argument("--rand-nodes", action="store_false", help="randomize node order in training")
parser.add_argument("--rule-override", action="store_false", help="rule-based override for symmetric relations")
parser.add_argument("--eval-dev", action='store_true', help="evaluate on dev set (otherwise - test)")
parser.add_argument("--dropout", type=float, default=0.0)
parser.add_argument("--nll", action='store_true', help="use negative log likelihood loss")
parser.add_argument("--no-log", action='store_true')
parser.add_argument("--early-stopping", action='store_true', help="stop if model hasn't improved in 3 epochs")
parser.add_argument("--v", type=int, default=1, help="verbosity")
parser.add_argument("--debug", action='store_true')
opts = parser.parse_args()
start_time = datetime.now()
# reporting
timeprint('graphs file = {}'.format(opts.input))
if opts.embeddings is not None:
timeprint('embeddings file = {}'.format(opts.embeddings))
else:
timeprint('embeddings size = {}'.format(opts.emb_size))
timeprint('association mode = {}'.format(opts.assoc_mode))
timeprint('negative samples = {}'.format(opts.neg_samp))
if opts.model is None:
timeprint('model file = {}'.format(opts.model_out))
if opts.nll:
timeprint('using negative log likelihood loss')
else:
timeprint('using margin loss')
if opts.no_log:
timeprint('no log file. timestamp for test: {}_{}' \
.format(start_time.date(), start_time.time()))
def init_ergm_cache(self):
"""
Computes ERGM features from scratch, populates cache members
"""
# edges
for r in self.relation_names:
edges = edge_count(self.graphs[r])
self.edge_counts[r] = edges
timeprint('populated edge cache')
# mutual edges
for i, n1 in enumerate(self.relation_names):
r1 = self.graphs[n1]
for j in range(i+1): # unordered, including self
n2 = self.relation_names[j]
r2 = self.graphs[n2]
mut_edges = mutual_edges(r1, r2)
self.mutual_edge_counts[(n1, n2)] = mut_edges
timeprint('populated mutual edge cache')
# directed triangles - iterate over R^2 + choose(r,3)/3 ordered relation triplets
eye = csr_eye(self.vocab_size)
for i,j,k in cyclic_triads(self.R):
n1, n2, n3 = self.relation_names[i], self.relation_names[j], self.relation_names[k]
r1, r2, r3 = self.graphs[n1], self.graphs[n2], self.graphs[n3]
dir_triangles = (r1 * r2 * r3).multiply(eye).sum()
if i == j and j == k: # each triangle was counted thrice, except self loops
self_loops = r1.diagonal().sum()
dir_triangles += 2 * self_loops
dir_triangles /=3
self.directed_triangle_counts[(n1, n2, n3)] = dir_triangles
timeprint('extracted directed triangle features')
# transitive closures - iterate over ordered relation triplets
# (also populate self.two_path_counts)
for n1, r1 in list(self.graphs.items()):
for n2, r2 in list(self.graphs.items()):
two_paths = r1 * r2
two_path_count = max([two_paths.sum(), sys.float_info.epsilon])
self.two_path_counts[(n1, n2)] = two_path_count
for n3, r3 in list(self.graphs.items()):
closures = two_paths.multiply(r3).sum() # pointwise
self.transitive_closure_counts[(n1, n2, n3)] = closures
timeprint('populated transitivity cache')
# 1-star cache for updates + self-2,3-stars
for n, g in list(self.graphs.items()):
self.in_degs[n] = g.sum(0) # numpy matrix
self.out_degs[n] = g.sum(1).transpose() # numpy matrix
osc = one_rel_star_counts(self.in_degs[n], self.out_degs[n])
self.in_one_star_counts[n] = osc['i1sc']
self.out_one_star_counts[n] = osc['o1sc']
self.in_two_star_counts[(n, n)] = osc['i2sc']
self.out_two_star_counts[(n, n)] = osc['o2sc']
self.in_three_star_counts[(n, n, n)] = osc['i3sc']
self.out_three_star_counts[(n, n, n)] = osc['o3sc']
self.in_one_p_star_counts[n] = osc['i1psc']
self.out_one_p_star_counts[n] = osc['o1psc']
self.in_two_p_star_counts[(n, n)] = osc['i2psc']
self.out_two_p_star_counts[(n, n)] = osc['o2psc']
self.in_three_p_star_counts[(n, n, n)] = osc['i3psc']
self.out_three_p_star_counts[(n, n, n)] = osc['o3psc']
timeprint('populated 1r-star cache')
# 2-stars
for n1, n2 in combinations(self.relation_names, 2):
tsc = two_rel_star_counts(self.in_degs[n1], self.out_degs[n1],\
self.in_degs[n2], self.out_degs[n2])
self.in_two_star_counts[(n1, n2)] = tsc['i2sc']
self.out_two_star_counts[(n1, n2)] = tsc['o2sc']
self.in_three_star_counts[(n1, n1, n2)] = tsc['i3sc112']
self.out_three_star_counts[(n1, n1, n2)] = tsc['o3sc112']
self.in_three_star_counts[(n1, n2, n2)] = tsc['i3sc122']
self.out_three_star_counts[(n1, n2, n2)] = tsc['o3sc122']
self.in_two_p_star_counts[(n1, n2)] = tsc['i2psc']
self.out_two_p_star_counts[(n1, n2)] = tsc['o2psc']
self.in_three_p_star_counts[(n1, n1, n2)] = tsc['i3psc112']
self.out_three_p_star_counts[(n1, n1, n2)] = tsc['o3psc112']
self.in_three_p_star_counts[(n1, n2, n2)] = tsc['i3psc122']
self.out_three_p_star_counts[(n1, n2, n2)] = tsc['o3psc122']
timeprint('populated 2r-star cache')
# 3-stars
for n1, n2, n3 in combinations(self.relation_names, 3):
ttsc = three_rel_star_counts(self.in_degs[n1], self.out_degs[n1],\
self.in_degs[n2], self.out_degs[n2],\
self.in_degs[n3], self.out_degs[n3])
self.in_three_star_counts[(n1, n2, n3)] = ttsc['i3sc']
self.out_three_star_counts[(n1, n2, n3)] = ttsc['o3sc']
self.in_three_p_star_counts[(n1, n2, n3)] = ttsc['i3psc']
self.out_three_p_star_counts[(n1, n2, n3)] = ttsc['o3psc']
timeprint('populated 3r-star cache')
def __init__(self, graphs, embeddings, assoc_mode=BILINEAR_MODE, reg=0.0, dropout=0.0,
no_assoc=False, model_path=None, ergm_path=None,
path_only_init=False):
"""
:param graphs: dictionary of {relation:CSR-format graph}s, node-aligned
:param embeddings: list of numpy array embeddings, indices aligned to nodes
:param model_path: optional path for files with pre-trained association model (read by super)
:param ergm_path: optional path for files with pre-trained model
:param path_only_init: model_path only used for initialization
"""
# input validation
AssociationModel.__init__(self, graphs, embeddings, assoc_mode, dropout, model_path=model_path)
# raw members
self.no_assoc = no_assoc
self.regularize = reg
# cache members
self.cache = {}
self.edge_counts = self.add_cache_dict('ec') # keys are single relations
self.mutual_edge_counts = self.add_cache_dict('mec') # keys are unordered relation pairs
self.two_path_counts = self.add_cache_dict('tpc') # keys are ordered relation pairs
self.transitive_closure_counts = self.add_cache_dict('tcc') # keys are ordered relation triplets
self.directed_triangle_counts = self.add_cache_dict('dtc') # keys are ordered relation triplets
self.in_degs = self.add_cache_dict('ins') # keys are single relations, values are big lists
self.out_degs = self.add_cache_dict('outs') # keys are single relations, values are big lists
self.in_one_star_counts = self.add_cache_dict('i1sc') # keys are single relations
self.out_one_star_counts = self.add_cache_dict('o1sc') # keys are single relations
self.in_two_star_counts = self.add_cache_dict('i2sc') # keys are unordered relation pairs
self.out_two_star_counts = self.add_cache_dict('o2sc') # keys are unordered relation pairs
self.in_three_star_counts = self.add_cache_dict('i3sc') # keys are unordered relation triplets
self.out_three_star_counts = self.add_cache_dict('o3sc') # keys are unordered relation triplets
# 'at least k' stars - 'one/two/three plus'
self.in_one_p_star_counts = self.add_cache_dict('i1psc') # keys are single relations
self.out_one_p_star_counts = self.add_cache_dict('o1psc') # keys are single relations
self.in_two_p_star_counts = self.add_cache_dict('i2psc') # keys are unordered relation pairs
self.out_two_p_star_counts = self.add_cache_dict('o2psc') # keys are unordered relation pairs
self.in_three_p_star_counts = self.add_cache_dict('i3psc') # keys are unordered relation triplets
self.out_three_p_star_counts = self.add_cache_dict('o3psc') # keys are unordered relation triplets
self.missing_node_indices = [] # updates during training (NOT SURE IF NEEDED)
timeprint('computing ERGM features...')
self.init_ergm_features() # populates self.feature_vals
timeprint('finished! computed {} features'.format(len(self.feature_vals)))
timeprint('{} non-zero features'.format(np.count_nonzero(list(self.feature_vals.values()))))
# documentationy again, for efficient updates
encountered_features = list(self.feature_vals.keys()) # canonical ordering from now on
if ergm_path is not None:
ergm_model_path = ergm_path
elif (model_path is not None) and (not path_only_init):
ergm_model_path = model_path
else:
ergm_model_path = None
if ergm_model_path is None:
self.feature_set = encountered_features
else:
self.feature_set = pickle.load(open(ergm_model_path + '.feats'))
assert sorted(self.feature_set) == sorted(encountered_features)
if ergm_model_path is None:
self.ergm_weights = self.model.add_parameters(len(self.feature_set))
if model_path is None and ergm_model_path is None:
# 'model_path is not None' is initialized in super()
# TODO support other association modes (affects downstream)
if self.no_assoc:
self.word_assoc_weights = {r:self.model.add_parameters((self.emb_dim, self.emb_dim), init=dy.ConstInitializer(0.0)) for r in self.relation_names}
else:
self.word_assoc_weights = {r:self.model.add_parameters((self.emb_dim, self.emb_dim)) for r in self.relation_names}
elif ergm_model_path is not None:
pc = dy.ParameterCollection()
dy.load(ergm_model_path + '.dyn', pc)
pc_list = pc.parameters_list()
i = 0
self.ergm_weights = pc_list[i]
if not path_only_init:
self.word_assoc_weights = {}
rel_order = self.relation_names
for r in rel_order:
i += 1
self.word_assoc_weights[r] = pc_list[i]
i += 1
assert i == len(pc_list),\
'{} relation params read but length is {}'.format(i, len(pc_list))
self.dy_score = self.ergm_score()
self.score = self.dy_score.scalar_value()
self.score_is_stale = False
timeprint('finished initialization. initial ERGM score = {}'.format(self.score))
LOWER = True
ALL_LEMMAS = True
def lemmas(s):
if ALL_LEMMAS:
name = '_'.join(s.lemma_names())
else:
name = s.lemma_names()[0]
if LOWER:
name = name.lower()
return name.split('_')
if __name__ == '__main__':
if len(sys.argv) < 3:
timeprint('usage: embed_from_words.py input_embs output_embs [WN prediction dataset]')
exit(1)
in_file = sys.argv[1]
# create target dataset
if len(sys.argv) > 3:
# third param is WN dataset
wn_vocab = load_prediction_dataset(sys.argv[3])[-1]
synsets = [wn.synset(w) for w in wn_vocab]
else:
synsets = list(wn.all_synsets())
timeprint('read {} synsets'.format(len(synsets)))
target_words = set()
timeprint('preparing target word dataset')
def node_iteration(rel, g, node, opts, assoc_model, trainer, log_file,
is_source):
"""
Perform one iteration of trying to score a node's neighbors above negative samples.
"""
# true instances likelihood
trues = targets(g, node) if is_source else sources(g, node)
side = '->' if is_source else '<-'
if len(trues) == 0: return 0.0
if opts.debug:
dy.renew_cg(immediate_compute=True, check_validity=True)
else:
dy.renew_cg()
# compute association score as dynet expression (can't do this above due to staleness)
true_scores = []
for tr in trues:
if is_source:
j_assoc_score = assoc_model.word_assoc_score(node, tr, rel)
else:
j_assoc_score = assoc_model.word_assoc_score(tr, node, rel)
if log_file is not None:
log_file.write('{} {}\tTRUE_{}\t{:.3e}\n'\
.format(node, side, tr, j_assoc_score.scalar_value()))
true_scores.append(j_assoc_score)
# false targets likelihood - negative sampling (uniform)
# collect negative samples
if opts.nll:
sample_scores = [[ts] for ts in true_scores]
else:
margins = []
neg_samples = [
np.random.choice(range(N)) for _ in range(opts.neg_samp * len(trues))
]
# remove source and true targets if applicable
for t in [node] + trues:
if t in neg_samples:
neg_samples.remove(t)
neg_samples.append(np.random.choice(range(N)))
for (i, ns) in enumerate(neg_samples):
# compute association score as dynet expression
if is_source:
ns_assoc_score = assoc_model.word_assoc_score(node, ns, rel)
else:
ns_assoc_score = assoc_model.word_assoc_score(ns, node, rel)
if log_file is not None:
log_file.write('{} {}\tNEG_{}\t{:.3e}\n'\
.format(node, side, ns, ns_assoc_score.scalar_value()))
corresponding_true = i // opts.neg_samp
if opts.nll:
sample_scores[corresponding_true].append(ns_assoc_score)
else:
# TODO maybe use dy.hinge()
ctt_score = true_scores[corresponding_true]
margin = ctt_score - ns_assoc_score
margins.append(dy.rectify(dy.scalarInput(1.0) - margin))
# compute overall loss
if opts.nll:
if len(sample_scores) == 0:
dy_loss = dy.scalarInput(0.0)
else:
dy_loss = dy.esum([
dy.pickneglogsoftmax(dy.concatenate(scrs), 0)
for scrs in sample_scores
])
else:
if len(margins) == 0:
dy_loss = dy.scalarInput(0.0)
else:
dy_loss = dy.esum(margins)
sc_loss = dy_loss.scalar_value()
if log_file is not None:
log_file.write('{}\tLOSS\t{:.3e}\n'\
.format(node, sc_loss))
# backprop and recompute score
if opts.v > 1:
timeprint('overall loss for relation {}, node {} as {} = {:.6f}'\
.format(rel, node, 'source' if is_source else 'target', sc_loss))
dy_loss.backward()
trainer.update()
return sc_loss
def macro_loops(opts,
ep_idx,
multi_graph,
trainer,
log_file,
synsets,
use_assoc=True):
"""
Passing over graph node by node, relation by relation.
Single update returned, based on importance sampling from entire graph.
:param opts: parameter dictionary from calling model
:param ep_idx: epoch index
:param multi_graph: trained data structure
:param trainer: dynet training module
:param log_file: log file location
:param synsets: synset name dictionary for reporting
:param use_assoc: include association component in scores
:return: node-iteration scores
"""
iteration_scores = []
iteration_scores.append(multi_graph.score)
N = multi_graph.vocab_size
timeprint('caching original graph features')
# report
if opts.v > 0:
timeprint('starting epoch {}'.format(ep_idx))
if not opts.rand_all:
# iterate over relations
graphs_order = list(multi_graph.graphs.keys())
if opts.rand_nodes:
dy.np.random.shuffle(graphs_order)
for rel in graphs_order:
# report
if opts.v > 0:
timeprint('starting loop over {}'.format(rel))
if opts.skip_symmetrics and rel in SYMMETRIC_RELATIONS:
timeprint('skipping symmetric relation {}'.format(rel))
continue
if rel == 'co_hypernym':
timeprint('skipping auxiliary co_hypernym relation')
continue
# compute target-wide association cache (no backprop)
if use_assoc:
assoc_cache = multi_graph.source_ranker_cache(rel)
else:
assoc_cache = np.zeros(
(multi_graph.word_assoc_weights[rel].shape()[0],
multi_graph.embeddings.shape()[1]))
timeprint('calculated association cache for {}'.format(rel))
# iterate over nodes:
node_order = list(range(N))
if opts.rand_nodes:
dy.np.random.shuffle(node_order)
for src_i in tqdm(node_order):
assoc_cache = macro_node_iteration(opts, multi_graph,
assoc_cache, trainer,
log_file, synsets, rel,
src_i, use_assoc)
multi_graph.rescore()
# total score = sum over all nodes
iteration_scores.append(multi_graph.score)
else:
# iterate randomly over <rel, node>-s iid
# rand_nodes implied
all_rels = list(multi_graph.graphs.keys())
if opts.skip_symmetrics:
rels = [r for r in all_rels if r not in SYMMETRIC_RELATIONS]
else:
rels = all_rels
if 'co_hypernym' in rels:
rels.remove('co_hypernym')
if use_assoc:
assoc_caches = {
rel: multi_graph.source_ranker_cache(rel)
for rel in rels
}
else:
assoc_caches = {
rel: np.zeros((multi_graph.word_assoc_weights[rel].shape()[0],
multi_graph.embeddings.shape()[1]))
for rel in rels
}
relnode_order = list(range(N * len(rels)))
dy.np.random.shuffle(relnode_order)
for idx in tqdm(relnode_order):
rel = rels[idx % len(rels)]
src_i = idx % N
assoc_caches[rel] = macro_node_iteration(opts, multi_graph,
assoc_caches[rel],
trainer, log_file,
synsets, rel, src_i,
use_assoc)
# only happens once in this setup
multi_graph.rescore()
# total score = sum over all nodes
iteration_scores.append(multi_graph.score)
return iteration_scores
parser.add_argument("--dropout", type=float, default=0.0)
parser.add_argument("--nll",
action='store_true',
help="use negative log likelihood loss")
parser.add_argument("--no-log", action='store_true')
parser.add_argument("--early-stopping",
action='store_true',
help="stop if model hasn't improved in 3 epochs")
parser.add_argument("--v", type=int, default=1, help="verbosity")
parser.add_argument("--debug", action='store_true')
opts = parser.parse_args()
start_time = datetime.now()
# reporting
timeprint('graphs file = {}'.format(opts.input))
if opts.embeddings is not None:
timeprint('embeddings file = {}'.format(opts.embeddings))
else:
timeprint('embeddings size = {}'.format(opts.emb_size))
timeprint('association mode = {}'.format(opts.assoc_mode))
timeprint('negative samples = {}'.format(opts.neg_samp))
if opts.model is None:
timeprint('model file = {}'.format(opts.model_out))
if opts.nll:
timeprint('using negative log likelihood loss')
else:
timeprint('using margin loss')
if opts.no_log:
timeprint('no log file. timestamp for test: {}_{}' \
.format(start_time.date(), start_time.time()))
def __init__(self,
graphs,
embeddings,
assoc_mode=BILINEAR_MODE,
reg=0.0,
dropout=0.0,
no_assoc=False,
model_path=None,
ergm_path=None,
path_only_init=False):
"""
:param graphs: dictionary of {relation:CSR-format graph}s, node-aligned
:param embeddings: list of numpy array embeddings, indices aligned to nodes
:param model_path: optional path for files with pre-trained association model (read by super)
:param ergm_path: optional path for files with pre-trained model
:param path_only_init: model_path only used for initialization
"""
# input validation
AssociationModel.__init__(self,
graphs,
embeddings,
assoc_mode,
dropout,
model_path=model_path)
# raw members
self.no_assoc = no_assoc
self.regularize = reg
# cache members
self.cache = {}
self.edge_counts = self.add_cache_dict(
'ec') # keys are single relations
self.mutual_edge_counts = self.add_cache_dict(
'mec') # keys are unordered relation pairs
self.two_path_counts = self.add_cache_dict(
'tpc') # keys are ordered relation pairs
self.transitive_closure_counts = self.add_cache_dict(
'tcc') # keys are ordered relation triplets
self.directed_triangle_counts = self.add_cache_dict(
'dtc') # keys are ordered relation triplets
self.in_degs = self.add_cache_dict(
'ins') # keys are single relations, values are big lists
self.out_degs = self.add_cache_dict(
'outs') # keys are single relations, values are big lists
self.in_one_star_counts = self.add_cache_dict(
'i1sc') # keys are single relations
self.out_one_star_counts = self.add_cache_dict(
'o1sc') # keys are single relations
self.in_two_star_counts = self.add_cache_dict(
'i2sc') # keys are unordered relation pairs
self.out_two_star_counts = self.add_cache_dict(
'o2sc') # keys are unordered relation pairs
self.in_three_star_counts = self.add_cache_dict(
'i3sc') # keys are unordered relation triplets
self.out_three_star_counts = self.add_cache_dict(
'o3sc') # keys are unordered relation triplets
# 'at least k' stars - 'one/two/three plus'
self.in_one_p_star_counts = self.add_cache_dict(
'i1psc') # keys are single relations
self.out_one_p_star_counts = self.add_cache_dict(
'o1psc') # keys are single relations
self.in_two_p_star_counts = self.add_cache_dict(
'i2psc') # keys are unordered relation pairs
self.out_two_p_star_counts = self.add_cache_dict(
'o2psc') # keys are unordered relation pairs
self.in_three_p_star_counts = self.add_cache_dict(
'i3psc') # keys are unordered relation triplets
self.out_three_p_star_counts = self.add_cache_dict(
'o3psc') # keys are unordered relation triplets
self.missing_node_indices = [
] # updates during training (NOT SURE IF NEEDED)
timeprint('computing ERGM features...')
self.init_ergm_features() # populates self.feature_vals
timeprint('finished! computed {} features'.format(
len(self.feature_vals)))
timeprint('{} non-zero features'.format(
np.count_nonzero(list(self.feature_vals.values()))))
# documentationy again, for efficient updates
encountered_features = list(
self.feature_vals.keys()) # canonical ordering from now on
if ergm_path is not None:
ergm_model_path = ergm_path
elif (model_path is not None) and (not path_only_init):
ergm_model_path = model_path
else:
ergm_model_path = None
if ergm_model_path is None:
self.feature_set = encountered_features
else:
self.feature_set = pickle.load(open(ergm_model_path + '.feats'))
assert sorted(self.feature_set) == sorted(encountered_features)
if ergm_model_path is None:
self.ergm_weights = self.model.add_parameters(len(
self.feature_set))
if model_path is None and ergm_model_path is None:
# 'model_path is not None' is initialized in super()
# TODO support other association modes (affects downstream)
if self.no_assoc:
self.word_assoc_weights = {
r: self.model.add_parameters((self.emb_dim, self.emb_dim),
init=dy.ConstInitializer(0.0))
for r in self.relation_names
}
else:
self.word_assoc_weights = {
r: self.model.add_parameters((self.emb_dim, self.emb_dim))
for r in self.relation_names
}
elif ergm_model_path is not None:
pc = dy.ParameterCollection()
dy.load(ergm_model_path + '.dyn', pc)
pc_list = pc.parameters_list()
i = 0
self.ergm_weights = pc_list[i]
if not path_only_init:
self.word_assoc_weights = {}
rel_order = self.relation_names
for r in rel_order:
i += 1
self.word_assoc_weights[r] = pc_list[i]
i += 1
assert i == len(pc_list),\
'{} relation params read but length is {}'.format(i, len(pc_list))
self.dy_score = self.ergm_score()
self.score = self.dy_score.scalar_value()
self.score_is_stale = False
timeprint('finished initialization. initial ERGM score = {}'.format(
self.score))
def init_ergm_cache(self):
"""
Computes ERGM features from scratch, populates cache members
"""
# edges
for r in self.relation_names:
edges = edge_count(self.graphs[r])
self.edge_counts[r] = edges
timeprint('populated edge cache')
# mutual edges
for i, n1 in enumerate(self.relation_names):
r1 = self.graphs[n1]
for j in range(i + 1): # unordered, including self
n2 = self.relation_names[j]
r2 = self.graphs[n2]
mut_edges = mutual_edges(r1, r2)
self.mutual_edge_counts[(n1, n2)] = mut_edges
timeprint('populated mutual edge cache')
# directed triangles - iterate over R^2 + choose(r,3)/3 ordered relation triplets
eye = csr_eye(self.vocab_size)
for i, j, k in cyclic_triads(self.R):
n1, n2, n3 = self.relation_names[i], self.relation_names[
j], self.relation_names[k]
r1, r2, r3 = self.graphs[n1], self.graphs[n2], self.graphs[n3]
dir_triangles = (r1 * r2 * r3).multiply(eye).sum()
if i == j and j == k: # each triangle was counted thrice, except self loops
self_loops = r1.diagonal().sum()
dir_triangles += 2 * self_loops
dir_triangles /= 3
self.directed_triangle_counts[(n1, n2, n3)] = dir_triangles
timeprint('extracted directed triangle features')
# transitive closures - iterate over ordered relation triplets
# (also populate self.two_path_counts)
for n1, r1 in list(self.graphs.items()):
for n2, r2 in list(self.graphs.items()):
two_paths = r1 * r2
two_path_count = max([two_paths.sum(), sys.float_info.epsilon])
self.two_path_counts[(n1, n2)] = two_path_count
for n3, r3 in list(self.graphs.items()):
closures = two_paths.multiply(r3).sum() # pointwise
self.transitive_closure_counts[(n1, n2, n3)] = closures
timeprint('populated transitivity cache')
# 1-star cache for updates + self-2,3-stars
for n, g in list(self.graphs.items()):
self.in_degs[n] = g.sum(0) # numpy matrix
self.out_degs[n] = g.sum(1).transpose() # numpy matrix
osc = one_rel_star_counts(self.in_degs[n], self.out_degs[n])
self.in_one_star_counts[n] = osc['i1sc']
self.out_one_star_counts[n] = osc['o1sc']
self.in_two_star_counts[(n, n)] = osc['i2sc']
self.out_two_star_counts[(n, n)] = osc['o2sc']
self.in_three_star_counts[(n, n, n)] = osc['i3sc']
self.out_three_star_counts[(n, n, n)] = osc['o3sc']
self.in_one_p_star_counts[n] = osc['i1psc']
self.out_one_p_star_counts[n] = osc['o1psc']
self.in_two_p_star_counts[(n, n)] = osc['i2psc']
self.out_two_p_star_counts[(n, n)] = osc['o2psc']
self.in_three_p_star_counts[(n, n, n)] = osc['i3psc']
self.out_three_p_star_counts[(n, n, n)] = osc['o3psc']
timeprint('populated 1r-star cache')
# 2-stars
for n1, n2 in combinations(self.relation_names, 2):
tsc = two_rel_star_counts(self.in_degs[n1], self.out_degs[n1],\
self.in_degs[n2], self.out_degs[n2])
self.in_two_star_counts[(n1, n2)] = tsc['i2sc']
self.out_two_star_counts[(n1, n2)] = tsc['o2sc']
self.in_three_star_counts[(n1, n1, n2)] = tsc['i3sc112']
self.out_three_star_counts[(n1, n1, n2)] = tsc['o3sc112']
self.in_three_star_counts[(n1, n2, n2)] = tsc['i3sc122']
self.out_three_star_counts[(n1, n2, n2)] = tsc['o3sc122']
self.in_two_p_star_counts[(n1, n2)] = tsc['i2psc']
self.out_two_p_star_counts[(n1, n2)] = tsc['o2psc']
self.in_three_p_star_counts[(n1, n1, n2)] = tsc['i3psc112']
self.out_three_p_star_counts[(n1, n1, n2)] = tsc['o3psc112']
self.in_three_p_star_counts[(n1, n2, n2)] = tsc['i3psc122']
self.out_three_p_star_counts[(n1, n2, n2)] = tsc['o3psc122']
timeprint('populated 2r-star cache')
# 3-stars
for n1, n2, n3 in combinations(self.relation_names, 3):
ttsc = three_rel_star_counts(self.in_degs[n1], self.out_degs[n1],\
self.in_degs[n2], self.out_degs[n2],\
self.in_degs[n3], self.out_degs[n3])
self.in_three_star_counts[(n1, n2, n3)] = ttsc['i3sc']
self.out_three_star_counts[(n1, n2, n3)] = ttsc['o3sc']
self.in_three_p_star_counts[(n1, n2, n3)] = ttsc['i3psc']
self.out_three_p_star_counts[(n1, n2, n3)] = ttsc['o3psc']
timeprint('populated 3r-star cache')
def remove_edge(self, src_idx, trg_idx, rel, update_feats=True, permanent=True,
caches=None, report_feat_diff=False):
"""
Removes edge from graph, updates cache and feature values
:param src_idx: index of source node from edge to remove
:param trg_idx: index of target node from edge to remove
:param rel: relation type
:param update_feats: flag for not updating all cache and features, to be deferred
:returns: if permanent=False, returns ergm score of removed-edge graph
"""
if permanent:
self.score_is_stale = True
cached_feats = None
cached_cache = None
else:
if caches is not None:
cached_cache = copy.deepcopy(caches[0])
cached_feats = caches[1]
else:
cached_cache = copy.deepcopy(self.cache)
cached_feats = copy.deepcopy(self.feature_vals)
update_feats=True # no other mode possible
# update cache members
# decrement edge count for rel
self.edge_counts[rel] -= 1
# pair cache members
for r,g in list(self.graphs.items()):
if rel == 'hypernym' and r == 'co_hypernym':
continue
# decrement mutual edge count for pairs with trg-src edges
if g[trg_idx, src_idx] == 1:
self.mutual_edge_counts[find_key(self.mutual_edge_counts, (rel, r))] -= 1
# decrement two-paths for x-src-trg and src-trg-y
self.two_path_counts[(r, rel)] -= self.in_degs[r][0,src_idx]
self.two_path_counts[(rel, r)] -= self.out_degs[r][0,trg_idx]
# triplet cache members
for r2, g2 in list(self.graphs.items()):
if rel == 'hypernym' and r2 == 'co_hypernym':
continue
# decrement transitive closures from two-paths src-x-trg
if self.out_degs[r][0,src_idx] > 0 and self.in_degs[r2][0,trg_idx] > 0:
r_r2_betweens = (g[src_idx] * g2[:,trg_idx]).sum()
self.transitive_closure_counts[(r, r2, rel)] -= r_r2_betweens
# decrement directed triangle count
if self.out_degs[r2][0,trg_idx] > 0 and self.in_degs[r][0,src_idx] > 0:
r_r2_cycles = (g2[trg_idx] * g[:,src_idx]).sum()
rs_key = find_cyclic_key(self.directed_triangle_counts, (r, rel, r2))
self.directed_triangle_counts[rs_key] -= r_r2_cycles
# decrement src's out_degree and trg's in_degree in rel and update all related caches
self.out_degs[rel][0,src_idx] -= 1
self.in_degs[rel][0,trg_idx] -= 1
if update_feats:
# recompute heavy cache updates from raw counts
self.update_stars_cache_from_edge(rel, src_idx, trg_idx, added=False)
# update features from caches
self.update_features()
if not permanent and report_feat_diff:
timeprint('changed feature values:')
diff_keys = [k for k in self.feature_vals if self.feature_vals[k] != cached_feats[k]]
if len(diff_keys) > 0:
print('\n'.join(['{}: from {} to {}'\
.format(k, cached_feats[k], self.feature_vals[k]) for k in diff_keys]))
if permanent:
# remove actual edge
self.graphs[rel][src_idx,trg_idx] = 0
else:
if rel == 'hypernym':
self.zero_all_feats('co_hypernym')
# prepare return value
ret = self.ergm_score()
# revert everything
self.reread_cache(cached_cache)
self.feature_vals = cached_feats
# return prepared score
return ret
def add_edge(self, src_idx, trg_idx, rel, permanent=False, caches=None, report_feat_diff=False):
"""
Uses cache to update feature values and produce score
:param src_idx: index of source node from edge to add
:param trg_idx: index of target node from edge to add
:param rel: relation type
:param permanent: True if node assignment to remain as is (inference mode, or restitution)
:param cache: optional - precomputed backup members (cache, features)
:return: new ergm score
"""
# back cache up
if caches is not None:
backup_cache = copy.deepcopy(caches[0])
backup_feats = caches[1]
elif not permanent:
backup_cache = copy.deepcopy(self.cache)
backup_feats = copy.deepcopy(self.feature_vals)
else:
backup_cache = None
backup_feats = None
# update cache members
# increment edge count for r
self.edge_counts[rel] += 1
# pair cache members
for r,g in list(self.graphs.items()):
if rel == 'hypernym' and r == 'co_hypernym':
continue
# increment mutual edge count for pairs with trg-src edges
if g[trg_idx, src_idx] == 1:
self.mutual_edge_counts[find_key(self.mutual_edge_counts, (rel, r))] += 1
# increment two-paths for x-src-trg and src-trg-y
self.two_path_counts[(r, rel)] += self.in_degs[r][0,src_idx]
self.two_path_counts[(rel, r)] += self.out_degs[r][0,trg_idx]
# triplet cache members
for r2, g2 in list(self.graphs.items()):
if rel == 'hypernym' and r2 == 'co_hypernym':
continue
# increment transitive closures from two-paths src-x-trg
if self.out_degs[r][0,src_idx] > 0 and self.in_degs[r2][0,trg_idx] > 0:
r_r2_betweens = (g[src_idx] * g2[:,trg_idx]).sum()
self.transitive_closure_counts[(r, r2, rel)] += r_r2_betweens
# increment directed triangle count
if self.out_degs[r2][0,trg_idx] > 0 and self.in_degs[r][0,src_idx] > 0:
r_r2_cycles = (g2[trg_idx] * g[:,src_idx]).sum()
rs_key = find_cyclic_key(self.directed_triangle_counts, (r, rel, r2))
self.directed_triangle_counts[rs_key] += r_r2_cycles
# increment src's out_degree and trg's in_degree in rel and update all related caches
self.out_degs[rel][0,src_idx] += 1
self.in_degs[rel][0,trg_idx] += 1
self.update_stars_cache_from_edge(rel, src_idx, trg_idx)
# update features from caches
self.update_features()
if rel == 'hypernym':
self.zero_all_feats('co_hypernym')
if report_feat_diff:
timeprint('changed feature values:')
diff_keys = [k for k in self.feature_vals if self.feature_vals[k] != backup_feats[k]]
print('\n'.join(['{}: from {} to {}'\
.format(k, backup_feats[k], self.feature_vals[k]) for k in diff_keys]))
# compute score for loss
ret = self.ergm_score()
if permanent:
# add actual edge
self.graphs[rel][src_idx,trg_idx] = 1
# update score
self.dy_score = ret
self.score = ret.scalar_value()
self.score_is_stale = False
else:
self.reread_cache(backup_cache)
self.feature_vals = backup_feats
return ret
def add_edge(self,
src_idx,
trg_idx,
rel,
permanent=False,
caches=None,
report_feat_diff=False):
"""
Uses cache to update feature values and produce score
:param src_idx: index of source node from edge to add
:param trg_idx: index of target node from edge to add
:param rel: relation type
:param permanent: True if node assignment to remain as is (inference mode, or restitution)
:param cache: optional - precomputed backup members (cache, features)
:return: new ergm score
"""
# back cache up
if caches is not None:
backup_cache = copy.deepcopy(caches[0])
backup_feats = caches[1]
elif not permanent:
backup_cache = copy.deepcopy(self.cache)
backup_feats = copy.deepcopy(self.feature_vals)
else:
backup_cache = None
backup_feats = None
# update cache members
# increment edge count for r
self.edge_counts[rel] += 1
# pair cache members
for r, g in list(self.graphs.items()):
if rel == 'hypernym' and r == 'co_hypernym':
continue
# increment mutual edge count for pairs with trg-src edges
if g[trg_idx, src_idx] == 1:
self.mutual_edge_counts[find_key(self.mutual_edge_counts,
(rel, r))] += 1
# increment two-paths for x-src-trg and src-trg-y
self.two_path_counts[(r, rel)] += self.in_degs[r][0, src_idx]
self.two_path_counts[(rel, r)] += self.out_degs[r][0, trg_idx]
# triplet cache members
for r2, g2 in list(self.graphs.items()):
if rel == 'hypernym' and r2 == 'co_hypernym':
continue
# increment transitive closures from two-paths src-x-trg
if self.out_degs[r][0, src_idx] > 0 and self.in_degs[r2][
0, trg_idx] > 0:
r_r2_betweens = (g[src_idx] * g2[:, trg_idx]).sum()
self.transitive_closure_counts[(r, r2,
rel)] += r_r2_betweens
# increment directed triangle count
if self.out_degs[r2][0, trg_idx] > 0 and self.in_degs[r][
0, src_idx] > 0:
r_r2_cycles = (g2[trg_idx] * g[:, src_idx]).sum()
rs_key = find_cyclic_key(self.directed_triangle_counts,
(r, rel, r2))
self.directed_triangle_counts[rs_key] += r_r2_cycles
# increment src's out_degree and trg's in_degree in rel and update all related caches
self.out_degs[rel][0, src_idx] += 1
self.in_degs[rel][0, trg_idx] += 1
self.update_stars_cache_from_edge(rel, src_idx, trg_idx)
# update features from caches
self.update_features()
if rel == 'hypernym':
self.zero_all_feats('co_hypernym')
if report_feat_diff:
timeprint('changed feature values:')
diff_keys = [
k for k in self.feature_vals
if self.feature_vals[k] != backup_feats[k]
]
print('\n'.join(['{}: from {} to {}'\
.format(k, backup_feats[k], self.feature_vals[k]) for k in diff_keys]))
# compute score for loss
ret = self.ergm_score()
if permanent:
# add actual edge
self.graphs[rel][src_idx, trg_idx] = 1
# update score
self.dy_score = ret
self.score = ret.scalar_value()
self.score_is_stale = False
else:
self.reread_cache(backup_cache)
self.feature_vals = backup_feats
return ret
parser.add_argument("--regularize", type=float, default=0.1)
parser.add_argument("--neg-samp", type=int, default=M3GM_DEFAULT_NEGS, help="number of negative samples")
parser.add_argument("--no-assoc-bp", action='store_true', help="do not backprop into association model")
parser.add_argument("--dropout", type=float, default=0.0,
help="dropout for association model only, set to 0.0 in no-assoc-bp mode")
# testing
parser.add_argument("--rule-override", type=bool, default=True, help="rule-based override for symmetric relations")
parser.add_argument("--rerank", type=int, default=100, help="number of top results to rerank")
opts = parser.parse_args()
# init
start_time = datetime.now()
# reporting
timeprint('graphs file = {}'.format(opts.input))
timeprint('embeddings file = {}'.format(opts.embeddings if opts.embeddings is not None \
else 'of size {}'.format(opts.emb_size)))
timeprint('association mode = {}'.format(opts.assoc_mode))
timeprint('reranker output file = {}'.format(opts.rerank_out))
if opts.model is None:
timeprint('model output file = {}, only init = {}'.format(opts.model_out, opts.model_only_init))
timeprint('epochs = {}'.format(opts.epochs))
timeprint('Adagrad learning rate = {}'.format(opts.learning_rate))
timeprint('neg-samp = {}'.format(opts.neg_samp))
timeprint('rand-nodes = {}'.format(opts.rand_nodes))
timeprint('dropout = {}'.format(opts.dropout))
timeprint('regularizer labmda = {}'.format(opts.regularize))
else:
timeprint('model file = {}, ergm model file = {}'.format(opts.model, opts.ergm_model))
if opts.ergm_model is not None:
def __init__(self,
graphs,
embeddings,
mode=TRANSLATIONAL_EMBED_MODE,
dropout=0.0,
model_path=None):
"""
:param graphs: dictionary of <relation:CSR-format graph>s, node-aligned
:param embeddings: list of numpy array embeddings, indices aligned to nodes
:param mode: mode of calculating association score, options: {}
""".format(MODES_STR)
# input validation
graph_sizes = list(set([g.shape[0] for g in list(graphs.values())]))
assert len(graph_sizes) == 1
assert len(embeddings) == graph_sizes[0], '{} != {}'.format(
len(embeddings), graph_sizes[0])
# raw members
self.graphs = {
canonicalize_name(k): g
for k, g in list(graphs.items())
}
self.mode = mode
# documenationy members
self.relation_names = sorted(self.graphs.keys())
if 'co_hypernym' in self.relation_names:
self.relation_names.remove('co_hypernym')
self.vocab_size = graph_sizes[0]
self.R = len(self.relation_names)
self.emb_dim = len(embeddings[0])
self.dropout = dropout
# model members
self.model = dy.Model()
# TODO consider using no_update param for embeddings
self.embeddings = self.model.add_lookup_parameters(
(self.vocab_size, self.emb_dim))
self.embeddings.init_from_array(embeddings)
# init association parameter
self.no_assoc = False # so can be overriden in inheritors
# first determine
if self.mode == BILINEAR_MODE: # full-rank bilinear matrix
assoc_dim = (self.emb_dim, self.emb_dim)
elif self.mode == DIAG_RANK1_MODE: # diagonal bilinear matrix + rank 1 matrix
# first row = diagonal
# second row = 'source factor'
# third row = 'target factor'
assoc_dim = (3, self.emb_dim)
elif self.mode == TRANSLATIONAL_EMBED_MODE: # additive relational vector
assoc_dim = self.emb_dim
elif self.mode == DISTMULT: # diagonal bilinear matrix
assoc_dim = self.emb_dim
else:
raise ValueError('unsupported mode: {}. allowed are {}'\
.format(self.mode, ', '.join(MODES_STR)))
# init actual parameter
self.word_assoc_weights = {
r: self.model.add_parameters(assoc_dim)
for r in self.relation_names
}
if model_path is not None:
self.model.populate(model_path + '.dyn')
timeprint('finished initialization for association model.')
def eval(prev_graphs, graphs, ergm, opts, N, log_file, rerank_file):
writing = log_file is not None
caches = (copy.deepcopy(ergm.cache),
copy.deepcopy(ergm.feature_vals))
rel_all_ranks = {} # for final results
rel_pre_ranks = {} # for improvement analysis
rel_erg_ranks = {} # for ergm-alone analysis
all_pre_ranks = []
all_all_ranks = []
all_erg_ranks = []
insts = Counter()
total_misses = Counter()
overrides = Counter()
rerank_ups = Counter()
rerank_downs = Counter()
erg_ups = Counter()
erg_downs = Counter()
rerank_diff = Counter()
erg_diff = Counter()
change_idx = 1
rels_order = list(graphs.items())
for rel, te_gr in rels_order:
if rel == 'co_hypernym':
continue
# set up
if writing:
timeprint('testing relation {}'.format(rel))
log_file.write('relation: {}\n'.format(rel))
# add incrementally, eval each edge, revert
tr_gr = prev_graphs[rel] # to filter known connections
s_assoc_cache = ergm.source_ranker_cache(rel)
t_assoc_cache = ergm.target_ranker_cache(rel)
override_rel = opts.rule_override and rel in SYMMETRIC_RELATIONS
all_ranks = []
pre_ranks = []
erg_ranks = []
if override_rel and writing:
log_file.write('RELATION OVERRIDE\n')
node_order = list(range(N)) # DO NOT RANDOMIZE THIS - NEED TO PREDICT BOTH SIDES
for node in tqdm(node_order):
s_trues, s_unch_loc_ranks, s_loc_gold_ranks, s_gold_reranked, s_gold_ergs, s_pls, change_idx = \
node_loop(change_idx, ergm, rel, node, s_assoc_cache,
caches, tr_gr, te_gr, override_rel, opts.rerank, True, log_file, rerank_file)
t_trues, t_unch_loc_ranks, t_loc_gold_ranks, t_gold_reranked, t_gold_ergs, t_pls, change_idx = \
node_loop(change_idx, ergm, rel, node, t_assoc_cache,
caches, tr_gr, te_gr, override_rel, opts.rerank, False, log_file, rerank_file)
total_trues = s_trues + t_trues
insts[rel] += (total_trues)
if override_rel:
overrides[rel] += total_trues
ulr = s_unch_loc_ranks + t_unch_loc_ranks
lgr = s_loc_gold_ranks + t_loc_gold_ranks
grr = s_gold_reranked + t_gold_reranked
ger = s_gold_ergs + t_gold_ergs
total_misses[rel] += (len(ulr))
pre_ranks.extend(lgr)
if override_rel:
erg_ranks.extend(lgr)
all_ranks.extend(lgr)
else:
all_ranks.extend(ulr + grr)
erg_ranks.extend(ulr + ger)
for pl in s_pls + t_pls:
if pl[3] < pl[2]:
rerank_ups[rel] += 1
if pl[3] > pl[2]:
rerank_downs[rel] += 1
if pl[4] < pl[2]:
erg_ups[rel] += 1
if pl[4] > pl[2]:
erg_downs[rel] += 1
rerank_diff[rel] += (pl[2] - pl[3])
erg_diff[rel] += (pl[2] - pl[4])
rel_all_ranks[rel] = all_ranks
rel_pre_ranks[rel] = pre_ranks
rel_erg_ranks[rel] = erg_ranks
all_all_ranks.extend(all_ranks)
all_pre_ranks.extend(pre_ranks)
all_erg_ranks.extend(erg_ranks)
if writing:
log_file.write('\nper relation:\n')
for rel in list(graphs.keys()):
if insts[rel] > 0 and insts[rel] - total_misses[rel] > 0:
log_file.write('\n{}:\n'.format(rel))
log_file.write('{} instances, {} misses\n'.format(insts[rel], total_misses[rel]))
log_file.write('reranks: {} up, {} down\n'.format(rerank_ups[rel], rerank_downs[rel]))
log_file.write('ERGM only: {} up, {} down\n'.format(erg_ups[rel], erg_downs[rel]))
log_file.write('rank diff: {}, ERGM only: {}\n'.format(rerank_diff[rel], erg_diff[rel]))
log_file.write('metrics: pre-rank\trerank\tERGM only\n')
log_file.write('average rank: {:.5f}\t{:.5f}\t{:.5f}\n'.format(np.average(rel_pre_ranks[rel]),
np.average(rel_all_ranks[rel]),
np.average(rel_erg_ranks[rel])))
log_file.write('mrr: {:.4f}\t{:.4f}\t{:.4f}\n'.format(mrr(rel_pre_ranks[rel]), mrr(rel_all_ranks[rel]),
mrr(rel_erg_ranks[rel])))
log_file.write(
'mq: {:.4f}\t{:.4f}\t{:.4f}\n'.format(mq(rel_pre_ranks[rel], N), mq(rel_all_ranks[rel], N),
mq(rel_erg_ranks[rel], N)))
log_file.write('h@100: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(rel_pre_ranks[rel], n=100),
h_at_n(rel_all_ranks[rel], n=100),
h_at_n(rel_erg_ranks[rel], n=100)))
log_file.write(
'h@10: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(rel_pre_ranks[rel]), h_at_n(rel_all_ranks[rel]),
h_at_n(rel_erg_ranks[rel])))
log_file.write('h@1: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(rel_pre_ranks[rel], n=1),
h_at_n(rel_all_ranks[rel], n=1),
h_at_n(rel_erg_ranks[rel], n=1)))
log_file.write('\ntotals:\n')
log_file.write('total number of instances: {}\n'.format(sum(insts.values())))
log_file.write('total misses: {}\n'.format(sum(total_misses.values())))
log_file.write('overrides: {}\n'.format(sum(overrides.values())))
log_file.write(
'rerank improvements: {}; regressions: {}\n'.format(sum(rerank_ups.values()), sum(rerank_downs.values())))
log_file.write(
'only ERGM improvements: {}; regressions: {}\n'.format(sum(erg_ups.values()), sum(erg_downs.values())))
log_file.write(
'total rank diffs: rerank {}, only ERGM {}\n'.format(sum(rerank_diff.values()), sum(erg_diff.values())))
log_file.write('metrics: pre-rank\trerank\tERGM only\n')
log_file.write(
'average rank: {:.5f}\t{:.5f}\t{:.5f}\n'.format(np.average(all_pre_ranks), np.average(all_all_ranks),
np.average(all_erg_ranks)))
log_file.write(
'mrr: {:.4f}\t{:.4f}\t{:.4f}\n'.format(mrr(all_pre_ranks), mrr(all_all_ranks), mrr(all_erg_ranks)))
log_file.write(
'mq: {:.4f}\t{:.4f}\t{:.4f}\n'.format(mq(all_pre_ranks, N), mq(all_all_ranks, N), mq(all_erg_ranks, N)))
log_file.write(
'h@100: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(all_pre_ranks, n=100), h_at_n(all_all_ranks, n=100),
h_at_n(all_erg_ranks, n=100)))
log_file.write('h@10: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(all_pre_ranks), h_at_n(all_all_ranks),
h_at_n(all_erg_ranks)))
log_file.write('h@1: {:.5f}\t{:.5f}\t{:.5f}\n'.format(h_at_n(all_pre_ranks, n=1), h_at_n(all_all_ranks, n=1),
h_at_n(all_erg_ranks, n=1)))
print('number of instances:', sum(insts.values()))
print('total misses:', sum(total_misses.values()))
print('overrides:', sum(overrides.values()))
print('average rank:', np.average(all_all_ranks))
print('mrr: {:.4f}'.format(mrr(all_all_ranks)))
print('mq:', mq(all_all_ranks, N))
print('h@100: {:.5f}'.format(h_at_n(all_all_ranks, n=100)))
print('h@10: {:.5f}'.format(h_at_n(all_all_ranks)))
print('h@1: {:.5f}'.format(h_at_n(all_all_ranks, n=1)))
return mrr(all_all_ranks), h_at_n(all_all_ranks, n=10), h_at_n(all_all_ranks, n=3), h_at_n(all_all_ranks, n=1)
def macro_node_iteration(opts, multi_graph, assoc_cache, trainer, log_file,
synsets, rel, src_i, use_assoc):
"""
One node-relation iteration in a macro-level pass over the multigraph
:param opts: parameter dictionary from calling model
:param multi_graph: trained data structure
:param assoc_cache: cache for association model
:param trainer: dynet training module
:param log_file: log file location
:param synsets: synset name dictionary for reporting
:param rel: relation type for iteration
:param src_i: source node ID for iteration
:param use_assoc: use association score model
:return: state of cache after iteration
"""
g = multi_graph.graphs[rel]
N = multi_graph.vocab_size
# set up iteration
if opts.debug:
dy.renew_cg(immediate_compute=True, check_validity=True)
else:
dy.renew_cg()
# keep existing score for all deltas
multi_graph.rescore()
score_with_all = multi_graph.dy_score
# report progress
perform_verbosity_steps = opts.v > 1 or (opts.v > 0 and src_i > 0
and src_i % 10 == 0)
if perform_verbosity_steps:
timeprint('iterating on node {}, {}, current score = {:.6f}'\
.format(src_i, synsets[src_i], score_with_all.scalar_value()))
# true targets scoring
true_targets = targets(g, src_i)
if len(true_targets) == 0:
# don't perform negative sampling without true targets
return assoc_cache
# compute log likelihood on targets
# each used to be multiplied by multi_graph.a_scale
target_assoc_scores = {
t: multi_graph.word_assoc_score(src_i, t, rel)
for t in true_targets
}
if opts.no_assoc_bp:
# turn into values to detach from computation graph
target_assoc_scores = {
t: t_as.value()
for t, t_as in list(target_assoc_scores.items())
}
target_scores = {
t: score_with_all + t_as
for t, t_as in list(target_assoc_scores.items())
}
# false targets scoring - importance sampling
# compute softmax over all false targets based on bilinear scores
if use_assoc:
assoc_sc = multi_graph.score_from_source_cache(assoc_cache, src_i)
neg_assocs = {
j: s
for j, s in enumerate(assoc_sc)
if j not in true_targets and j != src_i
}
else:
neg_assocs = {
j: 1.0
for j in range(N) if j not in true_targets and j != src_i
}
neg_probs = softmaxify(neg_assocs)
# collect negative samples
# TODO see if searchsorted can work here too (issue in dynet repo)
neg_samples = {t: [dy.np.random.choice(range(len(neg_assocs)), p=neg_probs)\
for _ in range(opts.neg_samp)]\
for t in true_targets} # sample without return?
# for reporting
if perform_verbosity_steps:
neg_sample_idcs = []
for negs in list(neg_samples.values()):
neg_sample_idcs.extend([list(neg_assocs.keys())[j] for j in negs])
# compute neg log likelihood on negative samples
margins = []
for t in true_targets:
t_score = target_scores[t]
negs = [list(neg_assocs.keys())[j] for j in neg_samples[t]]
# each used to be multiplied by multi_graph.a_scale
neg_assoc_scores = [
multi_graph.word_assoc_score(src_i, j, rel) for j in negs
]
if opts.no_assoc_bp:
# turn into values to detach from computation graph
neg_assoc_scores = [s.value() for s in neg_assoc_scores]
# prepare graph for pass
multi_graph.remove_edge(src_i, t, rel, permanent=True)
t_cache = (copy.deepcopy(multi_graph.cache),
copy.deepcopy(multi_graph.feature_vals))
for jas, j, origj in zip(neg_assoc_scores, negs, neg_samples[t]):
q_norm = 1.0 / neg_probs[origj]
g_score = multi_graph.add_edge(src_i,
j,
rel,
caches=t_cache,
report_feat_diff=opts.v > 1)
margins.append(
dy.rectify(g_score + jas + MARGIN - t_score) * q_norm)
log_file.write('{}\t{}\t{}\t{}\t{:.2e}\t{:.2e}\t{:.2e}\n'\
.format(rel, src_i, t, j, t_score.scalar_value(),
g_score.scalar_value(), jas if type(jas) == float else jas.value()))
# revert graph for next margin iteration
multi_graph.add_edge(src_i, t, rel, permanent=True)
node_loss = dy.esum(margins)
# backprop and recompute score
if perform_verbosity_steps:
timeprint('selected nodes {} with probabilities {}'\
.format(neg_sample_idcs, ['{:.2e}'.format(neg_probs[n]) for n in neg_samples]))
timeprint('overall {} loss = {:.6f}'\
.format('margin' if opts.margin_loss else 'neg log', node_loss.scalar_value()))
# record state for later reporting
pre_weights = multi_graph.ergm_weights.as_array()
pre_assoc = multi_graph.word_assoc_weights[rel].as_array()
# add regularization
if multi_graph.regularize > 0.0:
node_loss += multi_graph.regularize * dy.l2_norm(
dy.parameter(multi_graph.ergm_weights))
# perform actual learning
node_loss.backward()
trainer.update()
if perform_verbosity_steps:
post_weights = multi_graph.ergm_weights.as_array()
post_assoc = multi_graph.word_assoc_weights[rel].as_array()
w_diff = post_weights - pre_weights
a_diff = post_assoc - pre_assoc
timeprint('changed weights = {}'.format(len(w_diff.nonzero()[0])))
timeprint('changed pre_assoc = {}, norm {}'\
.format(len(a_diff.nonzero()[0]), np.linalg.norm(a_diff)))
# recompute assoc_cache columns for src_i and participating targets
if use_assoc and not opts.no_assoc_bp:
# TODO normalize embeddings?
return multi_graph.source_ranker_cache(rel)
return assoc_cache
def lemmas(s):
if ALL_LEMMAS:
name = '_'.join(s.lemma_names())
else:
name = s.lemma_names()[0]
if LOWER:
name = name.lower()
return name.split('_')
if __name__ == '__main__':
if len(sys.argv) < 3:
timeprint(
'usage: embed_from_words.py input_embs output_embs [WN prediction dataset]'
)
exit(1)
in_file = sys.argv[1]
# create target dataset
if len(sys.argv) > 3:
# third param is WN dataset
wn_vocab = load_prediction_dataset(sys.argv[3])[-1]
synsets = [wn.synset(w) for w in wn_vocab]
else:
synsets = list(wn.all_synsets())
timeprint('read {} synsets'.format(len(synsets)))
target_words = set()