def _compute_untagged_edges_and_transition_mat(self, full_graph, model):
logging.getLogger('main').info('Computing transition matrices...')
def _untag_edge(lexicon, edge):
source = lexicon.get_by_symstr(''.join(edge.source.word))[0]
target = lexicon.get_by_symstr(''.join(edge.target.word))[0]
rule = Rule(edge.rule.subst)
return GraphEdge(source, target, rule)
edge_prob = model.edges_prob(full_graph.edge_set)
edge_prob_ratios = edge_prob / (1 - edge_prob)
untagged_edge_set = EdgeSet(full_graph.lexicon)
T = len(self.tagset)
edge_ids_by_untagged_edge = []
for e_id, edge in enumerate(full_graph.edge_set):
untagged_edge = _untag_edge(full_graph.lexicon, edge)
if untagged_edge not in untagged_edge_set:
untagged_edge_set.add(untagged_edge)
edge_ids_by_untagged_edge.append(list())
ue_id = untagged_edge_set.get_id(untagged_edge)
edge_ids_by_untagged_edge[ue_id].append(e_id)
edge_tr_mat = []
for ue_id, e_ids in tqdm.tqdm(enumerate(edge_ids_by_untagged_edge), \
total=len(edge_ids_by_untagged_edge)):
tr_array = np.zeros((T, T))
for e_id in e_ids:
edge = full_graph.edge_set[e_id]
t1_id = self.tag_idx[edge.rule.tag_subst[0]]
t2_id = self.tag_idx[edge.rule.tag_subst[1]]
tr_array[t1_id, t2_id] = edge_prob_ratios[e_id]
if ue_id != len(edge_tr_mat):
raise Exception('Inconsistent untagged edge IDs!')
edge_tr_mat.append(csr_matrix(tr_array))
return untagged_edge_set, edge_tr_mat
def _compute_leaf_prob(self):
logging.getLogger('main').info('Computing leaf probabilities...')
self.leaf_prob = np.ones((len(self.lexicon), len(self.tagset)),
dtype=np.float64)
edge_set = EdgeSet(lexicon)
def _empty_edge_set(edge_set):
lexicon = edge_set.lexicon
n = len(edge_set)
probs = 1 - self.model.edges_prob(edge_set)
for e_id, edge in enumerate(edge_set):
word = lexicon.get_by_symstr(''.join(edge.source.word))[0]
w_id = lexicon.get_id(word)
t_id = self.tag_idx[edge.source.tag]
self.leaf_prob[w_id, t_id] *= probs[e_id]
edge_set = EdgeSet(lexicon)
print(n)
return edge_set
lexicon_tr = self.lexicon.to_fst()
lexicon_tr.concatenate(FST.generator(self.tagset))
rules_tr = self.model.rule_set.to_fst()
tr = hfst.HfstTransducer(lexicon_tr)
tr.compose(rules_tr)
tr.determinize()
tr.minimize()
FST.save_transducer(tr, 'tr.fsm')
tr_path = full_path('tr.fsm')
cmd = ['hfst-fst2strings', tr_path]
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
universal_newlines=True,
bufsize=1)
while True:
line = p.stdout.readline().strip()
if line:
w1, w2 = line.split(':')
n1 = LexiconEntry(w1)
n2 = LexiconEntry(w2)
rules = extract_all_rules(n1, n2)
for rule in rules:
if rule in rule_set:
edge_set.add(GraphEdge(n1, n2, rule))
else:
break
if len(edge_set) > 300000:
edge_set = _empty_edge_set(edge_set)
edge_set = _empty_edge_set(edge_set)
def run() -> None:
logging.getLogger('main').info('Loading lexicon...')
lexicon = Lexicon.load(shared.filenames['wordlist'])
logging.getLogger('main').info('Loading rules...')
rules_file = shared.filenames['rules-modsel']
if not file_exists(rules_file):
rules_file = shared.filenames['rules']
rule_set = RuleSet.load(rules_file)
edges_file = shared.filenames['graph-modsel']
if not file_exists(edges_file):
edges_file = shared.filenames['graph']
logging.getLogger('main').info('Loading the graph...')
edge_set = EdgeSet.load(edges_file, lexicon, rule_set)
full_graph = FullGraph(lexicon, edge_set)
if shared.config['General'].getboolean('supervised'):
full_graph.remove_isolated_nodes()
# full_graph.load_edges_from_file(graph_file)
# count rule frequencies in the full graph
# rule_freq = defaultdict(lambda: 0)
# for edge in full_graph.iter_edges():
# rule_freq[edge.rule] += 1
# initialize a PointModel
logging.getLogger('main').info('Initializing the model...')
model = ModelSuite(rule_set, lexicon=lexicon)
# model = PointModel()
# model.fit_rootdist(lexicon.entries())
# model.fit_ruledist(rule for (rule, domsize) in rules)
# for rule, domsize in rules:
# model.add_rule(rule, domsize, freq=rule_freq[rule])
softem(full_graph, model)
def compute_possible_edges(lexicon: Lexicon, rule_set: RuleSet) -> EdgeSet:
# build the transducer
lexicon_tr = lexicon.to_fst()
tag_seqs = extract_tag_symbols_from_rules(rule_set)
if tag_seqs:
lexicon_tr.concatenate(FST.generator(tag_seqs))
rules_tr = rule_set.to_fst()
tr = hfst.HfstTransducer(lexicon_tr)
tr.compose(rules_tr)
tr.determinize()
tr.minimize()
lexicon_tr.invert()
tr.compose(lexicon_tr)
tr.determinize()
tr.minimize()
FST.save_transducer(tr, 'tr.fsm')
tr_path = full_path('tr.fsm')
cmd = ['hfst-fst2strings', tr_path]
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
universal_newlines=True,
bufsize=1)
edge_set = EdgeSet(lexicon)
while True:
line = p.stdout.readline().strip()
if line:
w1, w2 = line.split(':')
w1_without_tag = re.sub(shared.compiled_patterns['tag'], '', w1)
w2_without_tag = re.sub(shared.compiled_patterns['tag'], '', w2)
if w1_without_tag != w2_without_tag:
n1 = LexiconEntry(w1)
n2 = LexiconEntry(w2)
rules = algorithms.align.extract_all_rules(n1, n2)
for rule in rules:
if rule in rule_set:
n1_wt = lexicon.get_by_symstr(w1_without_tag)[0]
n2_wt = lexicon.get_by_symstr(w2_without_tag)[0]
edge_set.add(GraphEdge(n1_wt, n2_wt, rule))
else:
break
return edge_set
def load_graph(filename, lexicon, threshold=0.0):
edge_set = EdgeSet(lexicon)
weights = []
rules = {}
for word_1, word_2, rule_str, edge_freq_str in read_tsv_file(filename):
try:
edge_freq = float(edge_freq_str)
if edge_freq < threshold:
continue
if rule_str not in rules:
rules[rule_str] = Rule.from_string(rule_str)
edge = GraphEdge(lexicon[word_1],
lexicon[word_2],
rules[rule_str],
weight=edge_freq)
edge_set.add(edge)
weights.append(edge_freq)
except ValueError:
pass
return FullGraph(lexicon, edge_set), np.array(weights)
def edges_cost(self, edge_set: EdgeSet) -> np.ndarray:
result = np.zeros(len(edge_set))
for rule, edge_ids in edge_set.get_edge_ids_by_rule().items():
rule_id = self.rule_set.get_id(rule)
freq_vector = np.array([edge_set[i].source.logfreq - \
edge_set[i].target.logfreq \
for i in edge_ids])
costs = -norm.logpdf(freq_vector, self.means[rule_id, ],
self.sdevs[rule_id, ])
result[tuple(edge_ids), ] = costs
return result
def load(model_type: str, filename: str, rule_set: RuleSet) -> EdgeModel:
if model_type == 'simple':
return SimpleEdgeModel.load(filename, rule_set)
elif model_type == 'neural':
lexicon = Lexicon.load(shared.filenames['wordlist'])
edge_set = \
EdgeSet.load(shared.filenames['graph'], lexicon, rule_set)
negex_sampler = NegativeExampleSampler(rule_set)
return NeuralEdgeModel.load(filename, rule_set, edge_set,
negex_sampler)
else:
raise UnknownModelTypeException('edge', model_type)
def edges_cost(self, edge_set: EdgeSet) -> np.ndarray:
result = np.zeros(len(edge_set))
for rule, edge_ids in edge_set.get_edge_ids_by_rule().items():
rule_id = self.rule_set.get_id(rule)
feature_matrix = np.vstack([edge_set[i].target.vec - \
edge_set[i].source.vec \
for i in edge_ids])
costs = -multivariate_normal.logpdf(feature_matrix,
self.means[rule_id, ],
np.diag(self.vars[rule_id, ]))
result[tuple(edge_ids), ] = costs
return result
def _empty_edge_set(edge_set):
lexicon = edge_set.lexicon
n = len(edge_set)
probs = 1 - self.model.edges_prob(edge_set)
for e_id, edge in enumerate(edge_set):
word = lexicon.get_by_symstr(''.join(edge.source.word))[0]
w_id = lexicon.get_id(word)
t_id = self.tag_idx[edge.source.tag]
self.leaf_prob[w_id, t_id] *= probs[e_id]
edge_set = EdgeSet(lexicon)
print(n)
return edge_set
def fit(self, edge_set: EdgeSet, weights: np.ndarray) -> None:
if self.means is None:
self.means = np.empty(len(self.rule_set))
if self.sdevs is None:
self.sdevs = np.empty(len(self.rule_set))
for rule, edge_ids in edge_set.get_edge_ids_by_rule().items():
edge_ids = tuple(edge_ids)
freq_vector = np.array([edge_set[i].source.logfreq - \
edge_set[i].target.logfreq \
for i in edge_ids])
self.fit_rule(self.rule_set.get_id(rule), freq_vector,
weights[edge_ids, ])
def fit(self, edge_set :EdgeSet, weights :np.ndarray) \
-> None:
if self.means is None:
self.means = np.empty((len(self.rule_set), self.dim))
if self.vars is None:
self.vars = np.empty((len(self.rule_set), self.dim))
for rule, edge_ids in edge_set.get_edge_ids_by_rule().items():
edge_ids = tuple(edge_ids)
feature_matrix = np.array([edge_set[i].target.vec - \
edge_set[i].source.vec \
for i in edge_ids])
self.fit_rule(self.rule_set.get_id(rule), feature_matrix,
weights[edge_ids, ])
def sample(self,
lexicon: Lexicon,
sample_size: int,
show_progressbar: bool = True) -> EdgeSet:
def _sample_process(rules: List[Rule], _output_fun: Callable[...,
None],
lexicon: Lexicon, sample_size: int) -> None:
transducers = [r.to_fst() for r in rules]
for tr in transducers:
tr.convert(hfst.ImplementationType.HFST_OL_TYPE)
seen_ids = set()
num = 0
while num < sample_size:
w_id = random.randrange(len(lexicon))
r_id = random.randrange(len(rules))
source = lexicon[w_id]
rule = rules[r_id]
lookup_results = \
sorted(list(map(lambda x: x[0].replace(hfst.EPSILON, ''),
transducers[r_id].lookup(source.symstr))))
if lookup_results:
t_id = random.randrange(len(lookup_results))
if (w_id, r_id, t_id) in seen_ids:
continue
seen_ids.add((w_id, r_id, t_id))
target = None
try:
target = LexiconEntry(lookup_results[t_id])
if target.symstr not in lexicon.items_by_symstr:
_output_fun(GraphEdge(source, target, rule))
num += 1
except Exception as e:
logging.getLogger('main').debug(\
'Exception during negative sampling: {}'.format(e))
num_processes = shared.config['NegativeExampleSampler']\
.getint('num_processes')
sample_size_per_proc = int(sample_size / num_processes)
edges_iter = \
parallel_execute(function=_sample_process,
data=list(self.rule_set),
num_processes=num_processes,
additional_args=(lexicon, sample_size_per_proc),
show_progressbar=show_progressbar,
progressbar_total = sample_size_per_proc * \
num_processes)
edge_set = EdgeSet(lexicon, edges_iter)
return edge_set
def run() -> None:
logging.getLogger('main').info('Loading lexicon...')
lexicon = Lexicon.load(shared.filenames['wordlist'])
logging.getLogger('main').info('Loading rules...')
rules_file = shared.filenames['rules-modsel']
if not file_exists(rules_file):
rules_file = shared.filenames['rules']
rule_set = RuleSet.load(rules_file)
edges_file = shared.filenames['graph-modsel']
if not file_exists(edges_file):
edges_file = shared.filenames['graph']
logging.getLogger('main').info('Loading the graph...')
edge_set = EdgeSet.load(edges_file, lexicon, rule_set)
full_graph = FullGraph(lexicon, edge_set)
# initialize a ModelSuite and save it
logging.getLogger('main').info('Initializing the model...')
model = ModelSuite(rule_set, lexicon=lexicon)
model.initialize(full_graph)
logging.getLogger('main').info('Saving the model...')
model.save()
def run() -> None:
logging.getLogger('main').info('Loading lexicon...')
lexicon = Lexicon.load(shared.filenames['wordlist'])
logging.getLogger('main').info('Loading rules...')
rule_set = RuleSet.load(shared.filenames['rules'])
logging.getLogger('main').info('Loading the graph...')
edge_set = EdgeSet.load(shared.filenames['graph'], lexicon, rule_set)
full_graph = FullGraph(lexicon, edge_set)
logging.getLogger('main').info('Initializing the model...')
model = ModelSuite(rule_set, lexicon=lexicon)
model.initialize(full_graph)
deleted_rules = set()
for iter_num in range(shared.config['modsel'].getint('iterations')):
sampler = MCMCGraphSampler(
full_graph, model,
shared.config['modsel'].getint('warmup_iterations'),
shared.config['modsel'].getint('sampling_iterations'))
sampler.add_stat('acc_rate', AcceptanceRateStatistic(sampler))
sampler.add_stat('edge_freq', EdgeFrequencyStatistic(sampler))
sampler.add_stat('exp_cost', ExpectedCostStatistic(sampler))
sampler.run_sampling()
# fit the model
edge_weights = sampler.stats['edge_freq'].value()
root_weights = np.ones(len(full_graph.lexicon))
for idx in range(edge_weights.shape[0]):
root_id = \
full_graph.lexicon.get_id(full_graph.edge_set[idx].target)
root_weights[root_id] -= edge_weights[idx]
model.fit(sampler.lexicon, sampler.edge_set, root_weights,
edge_weights)
# compute the rule statistics
freq, contrib = sampler.compute_rule_stats()
# determine the rules to delete
deleted_rules |= set(np.where(contrib < 0)[0])
logging.getLogger('main').info(\
'{} rules deleted.'.format(len(deleted_rules)))
# delete the edges with selected rules from the graph
edges_to_delete = []
for edge in full_graph.edges_iter():
if model.rule_set.get_id(edge.rule) in deleted_rules:
edges_to_delete.append(edge)
full_graph.remove_edges(edges_to_delete)
# deleting the rules is not necessary -- instead, save the reduced
# rule set at the end; fitting will be performed separately
logging.getLogger('main').info('Saving the graph...')
full_graph.edge_set.save(shared.filenames['graph-modsel'])
# remove the deleted rules from the rule set and save it
logging.getLogger('main').info('Saving the rule set...')
new_rule_set = RuleSet()
for i, rule in enumerate(rule_set):
if i not in deleted_rules:
new_rule_set.add(rule, rule_set.get_domsize(rule))
new_rule_set.save(shared.filenames['rules-modsel'])
def run() -> None:
logging.getLogger('main').info('Loading lexicon...')
lexicon = Lexicon.load(shared.filenames['wordlist'])
logging.getLogger('main').info('Loading rules...')
rules_file = shared.filenames['rules-modsel']
if not file_exists(rules_file):
rules_file = shared.filenames['rules']
rule_set = RuleSet.load(rules_file)
edges_file = shared.filenames['graph-modsel']
if not file_exists(edges_file):
edges_file = shared.filenames['graph']
logging.getLogger('main').info('Loading the graph...')
edge_set = EdgeSet.load(edges_file, lexicon, rule_set)
full_graph = FullGraph(lexicon, edge_set)
# initialize a ModelSuite
logging.getLogger('main').info('Loading the model...')
model = ModelSuite.load()
# setup the sampler
logging.getLogger('main').info('Setting up the sampler...')
sampler = MCMCGraphSamplerFactory.new(
full_graph,
model,
warmup_iter=shared.config['sample'].getint('warmup_iterations'),
sampling_iter=shared.config['sample'].getint('sampling_iterations'),
iter_stat_interval=shared.config['sample'].getint(
'iter_stat_interval'),
depth_cost=shared.config['Models'].getfloat('depth_cost'))
if shared.config['sample'].getboolean('stat_cost'):
sampler.add_stat('cost', stats.ExpectedCostStatistic(sampler))
if shared.config['sample'].getboolean('stat_acc_rate'):
sampler.add_stat('acc_rate', stats.AcceptanceRateStatistic(sampler))
if shared.config['sample'].getboolean('stat_iter_cost'):
sampler.add_stat('iter_cost', stats.CostAtIterationStatistic(sampler))
if shared.config['sample'].getboolean('stat_edge_freq'):
sampler.add_stat('edge_freq', stats.EdgeFrequencyStatistic(sampler))
if shared.config['sample'].getboolean('stat_undirected_edge_freq'):
sampler.add_stat('undirected_edge_freq',
stats.UndirectedEdgeFrequencyStatistic(sampler))
if shared.config['sample'].getboolean('stat_rule_freq'):
sampler.add_stat('freq', stats.RuleFrequencyStatistic(sampler))
if shared.config['sample'].getboolean('stat_rule_contrib'):
sampler.add_stat('contrib',
stats.RuleExpectedContributionStatistic(sampler))
# run sampling and print results
logging.getLogger('main').info('Running sampling...')
sampler.run_sampling()
sampler.summary()
sampler.save_root_costs('sample-root-costs.txt')
sampler.save_edge_costs('sample-edge-costs.txt')
# save paths to a file
pathlen = 0
with open_to_write('paths.txt') as fp:
for entry in lexicon:
root = sampler.branching.root(entry)
path = sampler.branching.path(root, entry)
path.reverse()
size = sampler.branching.subtree_size(root)
fp.write(' <- '.join([str(e) for e in path]) + \
' ({}, {})\n'.format(len(path), size))
pathlen += len(path)
logging.getLogger('main').debug('Average path length: {}'\
.format(pathlen / len(lexicon)))
# save rule frequency model fits to a file
if model.edge_frequency_model == 'lognormal':
with open_to_write('freqmodel.txt') as fp:
for r_id, rule in enumerate(model.rule_set):
write_line(fp, (rule, model.edge_frequency_model.means[r_id],
model.edge_frequency_model.sdevs[r_id]))
# count words at each depth in the graph
counts_per_depth = defaultdict(lambda: 0)
queue = [(word, 0) for word in lexicon \
if sampler.branching.parent(word) is None]
while queue:
(word, d) = queue.pop()
counts_per_depth[d] += 1
queue.extend([(word, d+1) \
for word in sampler.branching.successors(word)])
logging.getLogger('main').debug('Number of nodes per depth:')
for d, c in counts_per_depth.items():
logging.getLogger('main').debug('{} {}'.format(d, c))
def edges_cost(self, edge_set: EdgeSet) -> np.ndarray:
result = np.zeros(len(edge_set))
for rule, edge_ids in edge_set.get_edge_ids_by_rule().items():
result[edge_ids] = self._rule_appl_cost[self.rule_set.get_id(rule)]
return result
def edges_prob(self, edges: EdgeSet) -> np.ndarray:
result = np.zeros(len(edges))
for rule, edge_ids in edges.get_edge_ids_by_rule().items():
result[edge_ids] = self.rule_prob[self.rule_set.get_id(rule)]
return result
def analyze(self, target :LexiconEntry, compute_cost=True, **kwargs) \
-> List[GraphEdge]:
# TODO 1a. if predict_tag: get possible tags from the tag predictor
# 1. get possible sources for the given target
sources = set(sum([self.lexicon.get_by_symstr(word) \
for word, cost in self.fst.lookup(target.symstr)],
[]))
results = []
# 2. get possible (source, rule) pairs (extract rules) and score them
edge_set = EdgeSet(self.lexicon)
for source in sources:
rules = extract_all_rules(source, target)
for rule in rules:
if rule in self.model.rule_set:
if self.predict_vec:
target_pr = target.copy()
edge = GraphEdge(source, target_pr, rule)
target_pr.vec = self.model.predict_target_feature_vec(
edge)
edge_set.add(edge)
else:
edge_set.add(GraphEdge(source, target, rule))
# back-formation
if self.enable_back_formation and \
(self.max_results is None or len(edge_set) < self.max_results):
lookup_results = set()
for w, c in self.inv_rules_tr.lookup(target.symstr):
try:
lookup_results.add(unnormalize_word(\
re.sub(hfst.EPSILON, '', w)))
except Exception as e:
logging.getLogger('main').warning(str(e))
sources = []
for word in lookup_results:
try:
sources.append(LexiconEntry(word))
except Exception as e:
logging.getLogger('main').warning(str(e))
for source in sources:
rules = extract_all_rules(source, target)
for rule in rules:
if rule in self.model.rule_set:
edge_set.add(GraphEdge(source, target, rule))
# analysis as root
if self.include_roots:
edge_set.add(GraphEdge(None, target, None))
# scoring
# FIXME this is inefficient and may break on some model components
# that don't have the method .edge_cost()
for edge in edge_set:
edge.attr['cost'] = 0
if edge.source is not None:
edge.attr['cost'] += self.model.edge_cost(edge)
if edge.source not in self.lexicon:
edge.attr['cost'] += self.model.root_cost(edge.source)
else:
edge.attr['cost'] += self.model.root_cost(edge.target)
results = [edge for edge in edge_set]
# 4. sort the analyses according to the cost
results.sort(key=lambda r: r.attr['cost'])
if self.max_results is not None:
results = results[:self.max_results]
return results
def test_complete_sample(self) -> None:
'Test a sample consisting of all possible negative edges.'
words = [
'machen', 'macht', 'mache', 'Sachen', 'Sache', 'anwinkeln',
'anzuwinkeln'
]
rules = [\
':/en:t___:',
':/n:___:',
':/a:ä/:er___:',
':/:zu/:___:'
]
positive_edges = [\
('machen', 'macht', ':/en:t___:'),
('machen', 'mache', ':/n:___:'),
('Sachen', 'Sache', ':/n:___:'),
('anwinkeln', 'anzuwinkeln', ':/:zu/:___:'),
]
expected_negative_edges = [\
('Sachen', '{CAP}sacht', ':/en:t___:'),
('anwinkeln', 'anwinkel', ':/n:___:'),
('anzuwinkeln', 'anzuwinkel', ':/n:___:'),
('machen', 'mächener', ':/a:ä/:er___:'),
('macht', 'mächter', ':/a:ä/:er___:'),
('mache', 'mächeer', ':/a:ä/:er___:'),
('Sachen', '{CAP}sächener', ':/a:ä/:er___:'),
('Sache', '{CAP}sächeer', ':/a:ä/:er___:'),
('machen', 'mzuachen', ':/:zu/:___:'),
('machen', 'mazuchen', ':/:zu/:___:'),
('machen', 'maczuhen', ':/:zu/:___:'),
('machen', 'machzuen', ':/:zu/:___:'),
('machen', 'machezun', ':/:zu/:___:'),
('mache', 'mzuache', ':/:zu/:___:'),
('mache', 'mazuche', ':/:zu/:___:'),
('mache', 'maczuhe', ':/:zu/:___:'),
('mache', 'machzue', ':/:zu/:___:'),
('macht', 'mzuacht', ':/:zu/:___:'),
('macht', 'mazucht', ':/:zu/:___:'),
('macht', 'maczuht', ':/:zu/:___:'),
('macht', 'machzut', ':/:zu/:___:'),
('Sachen', '{CAP}zusachen', ':/:zu/:___:'),
('Sachen', '{CAP}szuachen', ':/:zu/:___:'),
('Sachen', '{CAP}sazuchen', ':/:zu/:___:'),
('Sachen', '{CAP}saczuhen', ':/:zu/:___:'),
('Sachen', '{CAP}sachzuen', ':/:zu/:___:'),
('Sachen', '{CAP}sachezun', ':/:zu/:___:'),
('Sache', '{CAP}zusache', ':/:zu/:___:'),
('Sache', '{CAP}szuache', ':/:zu/:___:'),
('Sache', '{CAP}sazuche', ':/:zu/:___:'),
('Sache', '{CAP}saczuhe', ':/:zu/:___:'),
('Sache', '{CAP}sachzue', ':/:zu/:___:'),
('anwinkeln', 'azunwinkeln', ':/:zu/:___:'),
('anwinkeln', 'anwzuinkeln', ':/:zu/:___:'),
('anwinkeln', 'anwizunkeln', ':/:zu/:___:'),
('anwinkeln', 'anwinzukeln', ':/:zu/:___:'),
('anwinkeln', 'anwinkzueln', ':/:zu/:___:'),
('anwinkeln', 'anwinkezuln', ':/:zu/:___:'),
('anwinkeln', 'anwinkelzun', ':/:zu/:___:'),
('anzuwinkeln', 'azunzuwinkeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuzuwinkeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzzuuwinkeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwzuinkeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwizunkeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwinzukeln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwinkzueln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwinkezuln', ':/:zu/:___:'),
('anzuwinkeln', 'anzuwinkelzun', ':/:zu/:___:')
]
expected_weights = {\
':/en:t___:' : 1.0,
':/n:___:' : 1.0,
':/a:ä/:er___:' : 1.0,
':/:zu/:___:' : 41/40 # the word "anzuzuwinkeln" can be
# derived in two different ways, so
# it is counted double in domsize
# computation, but sampled only once;
# such cases are very rare, so they
# shouldn't influence the weights much
}
lexicon = Lexicon(LexiconEntry(word) for word in words)
lex_fst = lexicon.to_fst()
rule_set = RuleSet()
for rule_str in rules:
rule = Rule.from_string(rule_str)
rule_set.add(rule, rule.compute_domsize(lex_fst))
edge_iter = (GraphEdge(lexicon[source], lexicon[target],
rule_set[rule]) \
for (source, target, rule) in positive_edges)
edge_set = EdgeSet(lexicon, edge_iter)
negex_sampler = NegativeExampleSampler(rule_set)
sample_size = len(expected_negative_edges)
sample = negex_sampler.sample(lexicon,
sample_size,
show_progressbar=False)
sample_weights = negex_sampler.compute_sample_weights(sample, edge_set)
self.assertEqual(rule_set.get_domsize(rule_set[0]), 2)
self.assertEqual(rule_set.get_domsize(rule_set[1]), 4)
self.assertEqual(rule_set.get_domsize(rule_set[2]), 5)
self.assertEqual(rule_set.get_domsize(rule_set[3]), 42)
self.longMessage = False
for edge in edge_set:
self.assertNotIn(edge,
sample,
msg='positive edge: {} in sample'.format(edge))
for source, target, rule in expected_negative_edges:
edge = GraphEdge(lexicon[source], LexiconEntry(target),
rule_set[rule])
self.assertIn(edge, sample, msg='{} not in sample'.format(edge))
self.longMessage = True
for i, edge in enumerate(sample):
self.assertAlmostEqual(sample_weights[i],
expected_weights[str(edge.rule)],
msg='for edge {}'.format(edge))