def test_mergedicsnodes(self):
tree = Tree.parse(
'(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4)'
'(VVPP 5)) (VAINF 6)) (VMFIN 3))',
parse_leaf=int)
assert str(mergediscnodes(splitdiscnodes(tree))) == (
'(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4) (VVPP 5)) '
'(VAINF 6)) (VMFIN 3))')
assert str(mergediscnodes(splitdiscnodes(tree, markorigin=True))) == (
'(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4) (VVPP 5)) '
'(VAINF 6)) (VMFIN 3))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))',
parse_leaf=int)
assert str(mergediscnodes(splitdiscnodes(
tree, markorigin=True))) == ('(S (X (A 0) (A 2)) (X (A 1) (A 3)))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))',
parse_leaf=int)
assert str(splitdiscnodes(tree, markorigin=True)) == (
'(S (X*0 (A 0)) (X*0 (A 1)) (X*1 (A 2)) (X*1 (A 3)))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))',
parse_leaf=int)
assert str(mergediscnodes(
splitdiscnodes(tree))) == ('(S (X (A 0) (A 1) (A 2) (A 3)))')
def test_mergedicsnodes(self):
tree = Tree.parse('(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4)'
'(VVPP 5)) (VAINF 6)) (VMFIN 3))', parse_leaf=int)
assert str(mergediscnodes(splitdiscnodes(tree))) == (
'(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4) (VVPP 5)) '
'(VAINF 6)) (VMFIN 3))')
assert str(mergediscnodes(splitdiscnodes(tree, markorigin=True))) == (
'(S (VP (VP (PP (APPR 0) (ART 1) (NN 2)) (CARD 4) (VVPP 5)) '
'(VAINF 6)) (VMFIN 3))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))', parse_leaf=int)
assert str(mergediscnodes(splitdiscnodes(tree, markorigin=True))) == (
'(S (X (A 0) (A 2)) (X (A 1) (A 3)))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))', parse_leaf=int)
assert str(splitdiscnodes(tree, markorigin=True)) == (
'(S (X*0 (A 0)) (X*0 (A 1)) (X*1 (A 2)) (X*1 (A 3)))')
tree = Tree.parse('(S (X (A 0) (A 2)) (X (A 1) (A 3)))', parse_leaf=int)
assert str(mergediscnodes(splitdiscnodes(tree))) == (
'(S (X (A 0) (A 1) (A 2) (A 3)))')
def test_splitdisc():
"""Verify that splitting and merging discontinuities gives the same
trees."""
from discodop.treebank import NegraCorpusReader
correct = wrong = 0
corpus = NegraCorpusReader('alpinosample.export')
for tree in corpus.trees().values():
if mergediscnodes(splitdiscnodes(tree)) == tree:
correct += 1
else:
wrong += 1
total = len(corpus.sents())
print('disc. split-merge: correct', correct, '=', 100. * correct / total, '%')
print('disc. split-merge: wrong', wrong, '=', 100. * wrong / total, '%')
assert wrong == 0
def test_splitdisc():
"""Verify that splitting and merging discontinuities gives the same
trees."""
from discodop.treebank import NegraCorpusReader
correct = wrong = 0
corpus = NegraCorpusReader('alpinosample.export')
for tree in corpus.trees().values():
if mergediscnodes(splitdiscnodes(tree)) == tree:
correct += 1
else:
wrong += 1
total = len(corpus.sents())
print('disc. split-merge: correct', correct, '=', 100. * correct / total, '%')
print('disc. split-merge: wrong', wrong, '=', 100. * wrong / total, '%')
assert wrong == 0
def getgrammars(trees, sents, stages, bintype, horzmarkov, vertmarkov, factor,
tailmarker, revmarkov, leftmostunary, rightmostunary, pospa, markhead,
fanout_marks_before_bin, testmaxwords, resultdir, numproc,
lexmodel, simplelexsmooth, top, relationalrealizational):
""" Apply binarization and read off the requested grammars. """
# fixme: this n should correspond to sentence id
tbfanout, n = treebankfanout(trees)
logging.info('treebank fan-out before binarization: %d #%d\n%s\n%s',
tbfanout, n, trees[n], ' '.join(sents[n]))
# binarization
begin = time.clock()
if fanout_marks_before_bin:
trees = [addfanoutmarkers(t) for t in trees]
if bintype == 'binarize':
bintype += ' %s h=%d v=%d %s' % (factor, horzmarkov, vertmarkov,
'tailmarker' if tailmarker else '')
for a in trees:
binarize(a, factor=factor, tailmarker=tailmarker,
horzmarkov=horzmarkov, vertmarkov=vertmarkov,
leftmostunary=leftmostunary, rightmostunary=rightmostunary,
reverse=revmarkov, pospa=pospa,
headidx=-1 if markhead else None,
filterfuncs=(relationalrealizational['ignorefunctions']
+ (relationalrealizational['adjunctionlabel'], ))
if relationalrealizational else ())
elif bintype == 'optimal':
trees = [Tree.convert(optimalbinarize(tree))
for n, tree in enumerate(trees)]
elif bintype == 'optimalhead':
trees = [Tree.convert(optimalbinarize(tree, headdriven=True,
h=horzmarkov, v=vertmarkov)) for n, tree in enumerate(trees)]
trees = [addfanoutmarkers(t) for t in trees]
logging.info('binarized %s cpu time elapsed: %gs',
bintype, time.clock() - begin)
logging.info('binarized treebank fan-out: %d #%d', *treebankfanout(trees))
trees = [canonicalize(a).freeze() for a in trees]
for n, stage in enumerate(stages):
if stage.split:
traintrees = [binarize(splitdiscnodes(Tree.convert(a),
stage.markorigin), childchar=':').freeze() for a in trees]
logging.info('splitted discontinuous nodes')
else:
traintrees = trees
if stage.mode.startswith('pcfg'):
assert tbfanout == 1 or stage.split
backtransform = None
if stage.dop:
if stage.usedoubledop:
# find recurring fragments in treebank,
# as well as depth 1 'cover' fragments
fragments = getfragments(traintrees, sents, numproc,
iterate=stage.iterate, complement=stage.complement)
xgrammar, backtransform, altweights = doubledop(
traintrees, fragments)
else: # DOP reduction
xgrammar, altweights = dopreduction(
traintrees, sents, packedgraph=stage.packedgraph)
nodes = sum(len(list(a.subtrees())) for a in traintrees)
if lexmodel and simplelexsmooth:
newrules = simplesmoothlexicon(lexmodel)
xgrammar.extend(newrules)
for weights in altweights.values():
weights.extend(w for _, w in newrules)
elif lexmodel:
xgrammar = smoothlexicon(xgrammar, lexmodel)
msg = grammarinfo(xgrammar)
rules, lexicon = write_lcfrs_grammar(
xgrammar, bitpar=stage.mode.startswith('pcfg'))
grammar = Grammar(rules, lexicon, start=top,
bitpar=stage.mode.startswith('pcfg'))
for name in altweights:
grammar.register(u'%s' % name, altweights[name])
with gzip.open('%s/%s.rules.gz' % (
resultdir, stage.name), 'wb') as rulesfile:
rulesfile.write(rules)
with codecs.getwriter('utf-8')(gzip.open('%s/%s.lex.gz' % (
resultdir, stage.name), 'wb')) as lexiconfile:
lexiconfile.write(lexicon)
logging.info('DOP model based on %d sentences, %d nodes, '
'%d nonterminals', len(traintrees), nodes, len(grammar.toid))
logging.info(msg)
if stage.estimator != 'dop1':
grammar.switch(u'%s' % stage.estimator)
_sumsto1 = grammar.testgrammar()
if stage.usedoubledop:
# backtransform keys are line numbers to rules file;
# to see them together do:
# $ paste <(zcat dop.rules.gz) <(zcat dop.backtransform.gz)
with codecs.getwriter('ascii')(gzip.open(
'%s/%s.backtransform.gz' % (resultdir, stage.name),
'w')) as out:
out.writelines('%s\n' % a for a in backtransform)
if n and stage.prune:
msg = grammar.getmapping(stages[n - 1].grammar,
striplabelre=None if stages[n - 1].dop
else re.compile(b'@.+$'),
neverblockre=re.compile(b'.+}<'),
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
else:
# recoverfragments() relies on this mapping to identify
# binarization nodes
msg = grammar.getmapping(None,
striplabelre=None,
neverblockre=re.compile(b'.+}<'),
splitprune=False, markorigin=False)
logging.info(msg)
elif n and stage.prune: # dop reduction
msg = grammar.getmapping(stages[n - 1].grammar,
striplabelre=None if stages[n - 1].dop
and not stages[n - 1].usedoubledop
else re.compile(b'@[-0-9]+$'),
neverblockre=re.compile(stage.neverblockre)
if stage.neverblockre else None,
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
if stage.mode == 'dop-rerank':
grammar.getrulemapping(stages[n - 1].grammar)
logging.info(msg)
# write prob models
np.savez_compressed('%s/%s.probs.npz' % (resultdir, stage.name),
**{name: mod for name, mod
in zip(grammar.modelnames, grammar.models)})
else: # not stage.dop
xgrammar = treebankgrammar(traintrees, sents)
logging.info('induced %s based on %d sentences',
('PCFG' if tbfanout == 1 or stage.split else 'PLCFRS'),
len(traintrees))
if stage.split or os.path.exists('%s/pcdist.txt' % resultdir):
logging.info(grammarinfo(xgrammar))
else:
logging.info(grammarinfo(xgrammar,
dump='%s/pcdist.txt' % resultdir))
if lexmodel and simplelexsmooth:
newrules = simplesmoothlexicon(lexmodel)
xgrammar.extend(newrules)
elif lexmodel:
xgrammar = smoothlexicon(xgrammar, lexmodel)
rules, lexicon = write_lcfrs_grammar(
xgrammar, bitpar=stage.mode.startswith('pcfg'))
grammar = Grammar(rules, lexicon, start=top,
bitpar=stage.mode.startswith('pcfg'))
with gzip.open('%s/%s.rules.gz' % (
resultdir, stage.name), 'wb') as rulesfile:
rulesfile.write(rules)
with codecs.getwriter('utf-8')(gzip.open('%s/%s.lex.gz' % (
resultdir, stage.name), 'wb')) as lexiconfile:
lexiconfile.write(lexicon)
_sumsto1 = grammar.testgrammar()
if n and stage.prune:
msg = grammar.getmapping(stages[n - 1].grammar,
striplabelre=None,
neverblockre=re.compile(stage.neverblockre)
if stage.neverblockre else None,
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
logging.info(msg)
logging.info('wrote grammar to %s/%s.{rules,lex%s}.gz', resultdir,
stage.name, ',backtransform' if stage.usedoubledop else '')
outside = None
if stage.getestimates == 'SX':
assert tbfanout == 1 or stage.split, 'SX estimate requires PCFG.'
logging.info('computing PCFG estimates')
begin = time.clock()
outside = getpcfgestimates(grammar, testmaxwords,
grammar.toid[trees[0].label])
logging.info('estimates done. cpu time elapsed: %gs',
time.clock() - begin)
np.savez('pcfgoutside.npz', outside=outside)
logging.info('saved PCFG estimates')
elif stage.useestimates == 'SX':
assert tbfanout == 1 or stage.split, 'SX estimate requires PCFG.'
assert stage.mode != 'pcfg', (
'estimates require agenda-based parser.')
outside = np.load('pcfgoutside.npz')['outside']
logging.info('loaded PCFG estimates')
if stage.getestimates == 'SXlrgaps':
logging.info('computing PLCFRS estimates')
begin = time.clock()
outside = getestimates(grammar, testmaxwords,
grammar.toid[trees[0].label])
logging.info('estimates done. cpu time elapsed: %gs',
time.clock() - begin)
np.savez('outside.npz', outside=outside)
logging.info('saved estimates')
elif stage.useestimates == 'SXlrgaps':
outside = np.load('outside.npz')['outside']
logging.info('loaded PLCFRS estimates')
stage.update(grammar=grammar, backtransform=backtransform,
outside=outside)
def getgrammars(trees, sents, stages, testmaxwords, resultdir,
numproc, lexmodel, simplelexsmooth, top):
"""Read off the requested grammars."""
tbfanout, n = treebank.treebankfanout(trees)
logging.info('binarized treebank fan-out: %d #%d', tbfanout, n)
for n, stage in enumerate(stages):
if stage.split:
traintrees = [treetransforms.binarize(
treetransforms.splitdiscnodes(
Tree.convert(a), stage.markorigin),
childchar=':', dot=True, ids=grammar.UniqueIDs()).freeze()
for a in trees]
logging.info('splitted discontinuous nodes')
else:
traintrees = trees
if stage.mode.startswith('pcfg'):
if tbfanout != 1 and not stage.split:
raise ValueError('Cannot extract PCFG from treebank '
'with discontinuities.')
backtransform = extrarules = None
if lexmodel and simplelexsmooth:
extrarules = lexicon.simplesmoothlexicon(lexmodel)
if stage.dop:
if stage.dop == 'doubledop':
(xgrammar, backtransform, altweights, fragments
) = grammar.doubledop(
traintrees, sents, binarized=stage.binarized,
iterate=stage.iterate, complement=stage.complement,
numproc=numproc, extrarules=extrarules)
# dump fragments
with codecs.getwriter('utf-8')(gzip.open('%s/%s.fragments.gz' %
(resultdir, stage.name), 'w')) as out:
out.writelines('%s\t%d\n' % (treebank.writetree(a, b, 0,
'bracket' if stage.mode.startswith('pcfg')
else 'discbracket').rstrip(), sum(c.values()))
for (a, b), c in fragments.items())
elif stage.dop == 'reduction':
xgrammar, altweights = grammar.dopreduction(
traintrees, sents, packedgraph=stage.packedgraph,
extrarules=extrarules)
else:
raise ValueError('unrecognized DOP model: %r' % stage.dop)
nodes = sum(len(list(a.subtrees())) for a in traintrees)
if lexmodel and not simplelexsmooth: # FIXME: altweights?
xgrammar = lexicon.smoothlexicon(xgrammar, lexmodel)
msg = grammar.grammarinfo(xgrammar)
rules, lex = grammar.write_lcfrs_grammar(
xgrammar, bitpar=stage.mode.startswith('pcfg'))
gram = Grammar(rules, lex, start=top,
bitpar=stage.mode.startswith('pcfg'),
binarized=stage.binarized)
for name in altweights:
gram.register(u'%s' % name, altweights[name])
with gzip.open('%s/%s.rules.gz' % (
resultdir, stage.name), 'wb') as rulesfile:
rulesfile.write(rules)
with codecs.getwriter('utf-8')(gzip.open('%s/%s.lex.gz' % (
resultdir, stage.name), 'wb')) as lexiconfile:
lexiconfile.write(lex)
logging.info('DOP model based on %d sentences, %d nodes, '
'%d nonterminals', len(traintrees), nodes, len(gram.toid))
logging.info(msg)
if stage.estimator != 'rfe':
gram.switch(u'%s' % stage.estimator)
logging.info(gram.testgrammar()[1])
if stage.dop == 'doubledop':
# backtransform keys are line numbers to rules file;
# to see them together do:
# $ paste <(zcat dop.rules.gz) <(zcat dop.backtransform.gz)
with codecs.getwriter('ascii')(gzip.open(
'%s/%s.backtransform.gz' % (resultdir, stage.name),
'w')) as out:
out.writelines('%s\n' % a for a in backtransform)
if n and stage.prune:
msg = gram.getmapping(stages[n - 1].grammar,
striplabelre=None if stages[n - 1].dop
else re.compile(b'@.+$'),
neverblockre=re.compile(b'.+}<'),
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
else:
# recoverfragments() relies on this mapping to identify
# binarization nodes
msg = gram.getmapping(None,
striplabelre=None,
neverblockre=re.compile(b'.+}<'),
splitprune=False, markorigin=False)
logging.info(msg)
elif n and stage.prune: # dop reduction
msg = gram.getmapping(stages[n - 1].grammar,
striplabelre=None if stages[n - 1].dop
and stages[n - 1].dop != 'doubledop'
else re.compile(b'@[-0-9]+$'),
neverblockre=re.compile(stage.neverblockre)
if stage.neverblockre else None,
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
if stage.mode == 'dop-rerank':
gram.getrulemapping(
stages[n - 1].grammar, re.compile(br'@[-0-9]+\b'))
logging.info(msg)
# write prob models
np.savez_compressed( # pylint: disable=no-member
'%s/%s.probs.npz' % (resultdir, stage.name),
**{name: mod for name, mod
in zip(gram.modelnames, gram.models)})
else: # not stage.dop
xgrammar = grammar.treebankgrammar(traintrees, sents,
extrarules=extrarules)
logging.info('induced %s based on %d sentences',
('PCFG' if tbfanout == 1 or stage.split else 'PLCFRS'),
len(traintrees))
if stage.split or os.path.exists('%s/pcdist.txt' % resultdir):
logging.info(grammar.grammarinfo(xgrammar))
else:
logging.info(grammar.grammarinfo(xgrammar,
dump='%s/pcdist.txt' % resultdir))
if lexmodel and not simplelexsmooth:
xgrammar = lexicon.smoothlexicon(xgrammar, lexmodel)
rules, lex = grammar.write_lcfrs_grammar(
xgrammar, bitpar=stage.mode.startswith('pcfg'))
gram = Grammar(rules, lex, start=top,
bitpar=stage.mode.startswith('pcfg'))
with gzip.open('%s/%s.rules.gz' % (
resultdir, stage.name), 'wb') as rulesfile:
rulesfile.write(rules)
with codecs.getwriter('utf-8')(gzip.open('%s/%s.lex.gz' % (
resultdir, stage.name), 'wb')) as lexiconfile:
lexiconfile.write(lex)
logging.info(gram.testgrammar()[1])
if n and stage.prune:
msg = gram.getmapping(stages[n - 1].grammar,
striplabelre=None,
neverblockre=re.compile(stage.neverblockre)
if stage.neverblockre else None,
splitprune=stage.splitprune and stages[n - 1].split,
markorigin=stages[n - 1].markorigin)
logging.info(msg)
logging.info('wrote grammar to %s/%s.{rules,lex%s}.gz',
resultdir, stage.name,
',backtransform' if stage.dop == 'doubledop' else '')
outside = None
if stage.estimates in ('SX', 'SXlrgaps'):
if stage.estimates == 'SX' and tbfanout != 1 and not stage.split:
raise ValueError('SX estimate requires PCFG.')
elif stage.mode != 'plcfrs':
raise ValueError('estimates require parser w/agenda.')
begin = time.clock()
logging.info('computing %s estimates', stage.estimates)
if stage.estimates == 'SX':
outside = estimates.getpcfgestimates(gram, testmaxwords,
gram.toid[trees[0].label])
elif stage.estimates == 'SXlrgaps':
outside = estimates.getestimates(gram, testmaxwords,
gram.toid[trees[0].label])
logging.info('estimates done. cpu time elapsed: %gs',
time.clock() - begin)
np.savez_compressed( # pylint: disable=no-member
'%s/%s.outside.npz' % (resultdir, stage.name),
outside=outside)
logging.info('saved %s estimates', stage.estimates)
elif stage.estimates:
raise ValueError('unrecognized value; specify SX or SXlrgaps.')
stage.update(grammar=gram, backtransform=backtransform,
outside=outside)
