def build_param(path='../util/proper.prm'):
param = readparam(path if path else None)
# param['CUTOFF_LEN'] = int(opts.get('--cutofflen', param['CUTOFF_LEN']))
# param['DISC_ONLY'] = '--disconly' in opts
# param['DEBUG'] = max(param['DEBUG'],
# '--verbose' in opts, 2 * ('--debug' in opts))
# param['TED'] |= '--ted' in opts
# param['LA'] |= '--la' in opts
# param['DEP'] = '--headrules' in opts
return param
def test_eval():
"""Simple sanity check; should give 100% score on all metrics."""
from discodop.treebank import READERS
from discodop.eval import Evaluator, readparam
gold = READERS['export']('alpinosample.export')
parses = READERS['export']('alpinosample.export')
goldtrees, goldsents, candsents = gold.trees(), gold.sents(), parses.sents()
evaluator = Evaluator(readparam(None))
for n, ctree in parses.trees().items():
evaluator.add(n, goldtrees[n], goldsents[n], ctree, candsents[n])
evaluator.breakdowns()
print(evaluator.summary())
def test_eval():
"""Simple sanity check; should give 100% score on all metrics."""
from discodop.treebank import READERS
from discodop.eval import Evaluator, readparam
gold = READERS['export']('alpinosample.export')
parses = READERS['export']('alpinosample.export')
goldtrees, goldsents, candsents = gold.trees(), gold.sents(), parses.sents()
evaluator = Evaluator(readparam(None))
for n, ctree in parses.trees().items():
evaluator.add(n, goldtrees[n], goldsents[n], ctree, candsents[n])
evaluator.breakdowns()
print(evaluator.summary())
from configparser import ConfigParser
from supertagging.data import SupertagParseDataset
config = ConfigParser()
config.read(argv[1])
data = SupertagParseDataset(f"{config['Corpus']['filename']}.train")
from discodop.tree import ParentedTree, Tree
from discodop.treetransforms import unbinarize, removefanoutmarkers
from discodop.eval import Evaluator, readparam
from discodop.lexgrammar import SupertagGrammar
grammar = load(open(f"{config['Corpus']['filename']}.grammar", "rb"))
i = 0
evaluator = Evaluator(readparam("proper.prm"))
for sentence in data:
words = tuple(t.text for t in sentence)
poss = tuple(t.get_tag("pos").value for t in sentence)
tags = tuple(((t.get_tag("supertag").value, 0.0), ) for t in sentence)
parses = grammar.parse(poss, tags, posmode=True)
try:
parse = next(parses)
except StopIteration:
leaves = (f"({p} {i})" for p, i in zip(poss, range(len(words))))
parse = ParentedTree(f"(NOPARSE {' '.join(leaves)})")
gold = ParentedTree(sentence.get_labels("tree")[0].value)
gold = ParentedTree.convert(
unbinarize(removefanoutmarkers(Tree.convert(gold))))
parse = ParentedTree.convert(
unbinarize(removefanoutmarkers(Tree.convert(parse))))
def set_eval_param(self, config: EvalParameters):
self.__grammar__.fallback_prob = config.fallbackprob
self.__evalparam__ = readparam(config.evalfilename)
self.__ktags__ = config.ktags
def startexp(
stages=(DEFAULTSTAGE, ), # see above
corpusfmt='export', # choices: export, discbracket, bracket
corpusdir='.',
# filenames may include globbing characters '*' and '?'.
traincorpus='alpinosample.export', trainencoding='utf-8',
testcorpus='alpinosample.export', testencoding='utf-8',
testmaxwords=40,
trainmaxwords=40,
trainnumsents=2,
testnumsents=1, # number of sentences to parse
skiptrain=True, # test set starts after training set
# (useful when they are in the same file)
skip=0, # number of sentences to skip from test corpus
punct=None, # choices: None, 'move', 'remove', 'root'
functions=None, # choices None, 'add', 'remove', 'replace'
morphology=None, # choices: None, 'add', 'replace', 'between'
transformations=None, # apply treebank transformations
# postagging: pass None to use tags from treebank.
postagging=None,
relationalrealizational=None, # do not apply RR-transform
headrules=None, # rules for finding heads of constituents
bintype='binarize', # choices: binarize, optimal, optimalhead
factor='right',
revmarkov=True,
v=1,
h=2,
pospa=False, # when v > 1, add parent annotation to POS tags?
markhead=False, # prepend head to siblings
leftmostunary=True, # start binarization with unary node
rightmostunary=True, # end binarization with unary node
tailmarker='', # with headrules, head is last node and can be marked
fanout_marks_before_bin=False,
evalparam='proper.prm', # EVALB-style parameter file
quiet=False, reallyquiet=False, # quiet=no per sentence results
numproc=1, # increase to use multiple CPUs; None: use all CPUs.
resultdir='results',
rerun=False):
""" Execute an experiment. """
assert bintype in ('optimal', 'optimalhead', 'binarize')
if postagging is not None:
assert set(postagging).issubset({'method', 'model',
'unknownthreshold', 'openclassthreshold', 'simplelexsmooth'})
if postagging['method'] == 'unknownword':
assert postagging['model'] in ('4', '6', 'base')
assert postagging['unknownthreshold'] >= 1
assert postagging['openclassthreshold'] >= 0
else:
assert postagging['method'] in ('treetagger', 'stanford')
if rerun:
assert os.path.exists(resultdir), (
'Directory %r does not exist.'
'--rerun requires a directory '
'with the grammar(s) of a previous experiment.'
% resultdir)
else:
assert not os.path.exists(resultdir), (
'Directory %r exists.\n'
'Use --rerun to parse with existing grammar '
'and overwrite previous results.' % resultdir)
os.mkdir(resultdir)
# Log everything, and send it to stderr, in a format with just the message.
formatstr = '%(message)s'
if reallyquiet:
logging.basicConfig(level=logging.WARNING, format=formatstr)
elif quiet:
logging.basicConfig(level=logging.INFO, format=formatstr)
else:
logging.basicConfig(level=logging.DEBUG, format=formatstr)
# also log to a file
fileobj = logging.FileHandler(filename='%s/output.log' % resultdir)
#fileobj.setLevel(logging.INFO)
fileobj.setLevel(logging.DEBUG)
fileobj.setFormatter(logging.Formatter(formatstr))
logging.getLogger('').addHandler(fileobj)
corpusreader = getreader(corpusfmt)
if not rerun:
corpus = corpusreader(corpusdir, traincorpus, encoding=trainencoding,
headrules=headrules, headfinal=True, headreverse=False,
punct=punct, functions=functions, morphology=morphology)
logging.info('%d sentences in training corpus %s/%s',
len(corpus.parsed_sents()), corpusdir, traincorpus)
if isinstance(trainnumsents, float):
trainnumsents = int(trainnumsents * len(corpus.sents()))
trees = list(corpus.parsed_sents().values())[:trainnumsents]
sents = list(corpus.sents().values())[:trainnumsents]
if transformations:
trees = [transform(tree, sent, transformations)
for tree, sent in zip(trees, sents)]
if relationalrealizational:
trees = [rrtransform(tree, **relationalrealizational)[0]
for tree in trees]
train_tagged_sents = [[(word, tag) for word, (_, tag)
in zip(sent, sorted(tree.pos()))]
for tree, sent in zip(trees, sents)]
blocks = list(corpus.blocks().values())[:trainnumsents]
assert trees, 'training corpus should be non-empty'
logging.info('%d training sentences before length restriction',
len(trees))
trees, sents, blocks = zip(*[sent for sent in zip(trees, sents, blocks)
if len(sent[1]) <= trainmaxwords])
logging.info('%d training sentences after length restriction <= %d',
len(trees), trainmaxwords)
testset = corpusreader(corpusdir, testcorpus, encoding=testencoding,
punct=punct, morphology=morphology, functions=functions)
gold_sents = testset.tagged_sents()
test_parsed_sents = testset.parsed_sents()
if skiptrain:
skip += trainnumsents
logging.info('%d sentences in test corpus %s/%s',
len(testset.parsed_sents()), corpusdir, testcorpus)
logging.info('%d test sentences before length restriction',
len(list(gold_sents)[skip:skip + testnumsents]))
lexmodel = None
test_tagged_sents = gold_sents
if postagging and postagging['method'] in ('treetagger', 'stanford'):
if postagging['method'] == 'treetagger':
# these two tags are never given by tree-tagger,
# so collect words whose tag needs to be overriden
overridetags = ('PTKANT', 'PIDAT')
elif postagging['method'] == 'stanford':
overridetags = ('PTKANT', )
taglex = defaultdict(set)
for sent in train_tagged_sents:
for word, tag in sent:
taglex[word].add(tag)
overridetagdict = {tag:
{word for word, tags in taglex.items() if tags == {tag}}
for tag in overridetags}
tagmap = {'$(': '$[', 'PAV': 'PROAV'}
sents_to_tag = OrderedDict((a, b) for a, b
in islice(gold_sents.items(), skip, skip + testnumsents)
if len(b) <= testmaxwords),
test_tagged_sents = externaltagging(postagging['method'],
postagging['model'], sents_to_tag, overridetagdict, tagmap)
# give these tags to parser
usetags = True
elif postagging and postagging['method'] == 'unknownword' and not rerun:
postagging['unknownwordfun'] = getunknownwordfun(postagging['model'])
# get smoothed probalities for lexical productions
lexresults, msg = getunknownwordmodel(
train_tagged_sents, postagging['unknownwordfun'],
postagging['unknownthreshold'],
postagging['openclassthreshold'])
logging.info(msg)
simplelexsmooth = postagging['simplelexsmooth']
if simplelexsmooth:
lexmodel = lexresults[2:8]
else:
lexmodel, msg = getlexmodel(*lexresults)
logging.info(msg)
# NB: knownwords are all words in training set, lexicon is the subset
# of words that are above the frequency threshold.
# for training purposes we work with the subset, at test time we exploit
# the full set of known words from the training set.
sigs, knownwords, lexicon = lexresults[:3]
postagging['sigs'], postagging['lexicon'] = sigs, knownwords
# replace rare train words with signatures
sents = replaceraretrainwords(train_tagged_sents,
postagging['unknownwordfun'], lexicon)
# make sure gold POS tags are not given to parser
usetags = False
elif postagging and postagging['method'] == 'unknownword' and rerun:
usetags = False
else:
simplelexsmooth = False
# give gold POS tags to parser
usetags = True
# 0: test sentences as they should be handed to the parser,
# 1: gold trees for evaluation purposes
# 2: gold sentence because test sentences may be mangled by unknown word
# model
# 3: blocks from treebank file to reproduce the relevant part of the
# original treebank verbatim.
testset = OrderedDict((a, (test_tagged_sents[a], test_parsed_sents[a],
gold_sents[a], block)) for a, block
in islice(testset.blocks().items(), skip, skip + testnumsents)
if len(test_tagged_sents[a]) <= testmaxwords)
assert test_tagged_sents, 'test corpus should be non-empty'
logging.info('%d test sentences after length restriction <= %d',
len(testset), testmaxwords)
if rerun:
trees = []
sents = []
toplabels = {tree.label for tree in trees} | {
test_parsed_sents[n].label for n in testset}
assert len(toplabels) == 1, 'expected unique ROOT label: %r' % toplabels
top = toplabels.pop()
if rerun:
readgrammars(resultdir, stages, postagging, top)
else:
logging.info('read training & test corpus')
getgrammars(trees, sents, stages, bintype, h, v, factor, tailmarker,
revmarkov, leftmostunary, rightmostunary, pospa, markhead,
fanout_marks_before_bin, testmaxwords, resultdir, numproc,
lexmodel, simplelexsmooth, top, relationalrealizational)
evalparam = evalmod.readparam(evalparam)
evalparam['DEBUG'] = -1
evalparam['CUTOFF_LEN'] = 40
deletelabel = evalparam.get('DELETE_LABEL', ())
deleteword = evalparam.get('DELETE_WORD', ())
begin = time.clock()
parser = Parser(stages, transformations=transformations,
tailmarker=tailmarker, postagging=postagging if postagging
and postagging['method'] == 'unknownword' else None,
relationalrealizational=relationalrealizational)
results = doparsing(parser=parser, testset=testset, resultdir=resultdir,
usetags=usetags, numproc=numproc, deletelabel=deletelabel,
deleteword=deleteword, corpusfmt=corpusfmt, morphology=morphology)
if numproc == 1:
logging.info('time elapsed during parsing: %gs', time.clock() - begin)
for result in results[0]:
nsent = len(result.parsetrees)
header = (' ' + result.name.upper() + ' ').center(35, '=')
evalsummary = evalmod.doeval(OrderedDict((a, b.copy(True))
for a, b in test_parsed_sents.items()), gold_sents,
result.parsetrees, test_tagged_sents if usetags else gold_sents,
evalparam)
coverage = 'coverage: %s = %6.2f' % (
('%d / %d' % (nsent - result.noparse, nsent)).rjust(
25 if any(len(a) > evalparam['CUTOFF_LEN']
for a in gold_sents.values()) else 14),
100.0 * (nsent - result.noparse) / nsent)
logging.info('\n'.join(('', header, evalsummary, coverage)))
return top
def startexp(
stages=(parser.DictObj(parser.DEFAULTSTAGE), ), # see parser module
corpusfmt='export', # choices: export, (disc)bracket, alpino, tiger
traincorpus=parser.DictObj(DEFAULTS['traincorpus']),
testcorpus=parser.DictObj(DEFAULTS['testcorpus']),
binarization=parser.DictObj(DEFAULTS['binarization']),
removeempty=False, # whether to remove empty terminals
ensureroot=None, # ensure every tree has a root node with this label
punct=None, # choices: None, 'move', 'remove', 'root'
functions=None, # choices None, 'add', 'remove', 'replace'
morphology=None, # choices: None, 'add', 'replace', 'between'
transformations=None, # apply treebank transformations
postagging=None, # postagging: pass None to use tags from treebank.
relationalrealizational=None, # do not apply RR-transform
evalparam='proper.prm', # EVALB-style parameter file
verbosity=2,
numproc=1, # increase to use multiple CPUs; None: use all CPUs.
resultdir='results',
rerun=False):
"""Execute an experiment."""
if rerun:
if not os.path.exists(resultdir):
raise ValueError('Directory %r does not exist.\n--rerun requires a'
' directory with the grammar(s) of a previous experiment.'
% resultdir)
else:
if os.path.exists(resultdir):
raise ValueError('Directory %r exists.\n'
'Use --rerun to parse with existing grammar '
'and overwrite previous results.' % resultdir)
os.mkdir(resultdir)
# Log everything, and send it to stderr, in a format with just the message.
formatstr = '%(message)s'
if verbosity == 0:
logging.basicConfig(level=logging.WARNING, format=formatstr)
elif verbosity == 1:
logging.basicConfig(level=logging.INFO, format=formatstr)
elif verbosity == 2:
logging.basicConfig(level=logging.DEBUG, format=formatstr)
elif 3 <= verbosity <= 4:
logging.basicConfig(level=5, format=formatstr)
else:
raise ValueError('verbosity should be >= 0 and <= 4. ')
# also log to a file
fileobj = logging.FileHandler(filename='%s/output.log' % resultdir)
fileobj.setLevel(logging.DEBUG)
fileobj.setFormatter(logging.Formatter(formatstr))
logging.getLogger('').addHandler(fileobj)
if not rerun:
trees, sents, train_tagged_sents = loadtraincorpus(
corpusfmt, traincorpus, binarization, punct, functions,
morphology, removeempty, ensureroot, transformations,
relationalrealizational)
elif isinstance(traincorpus.numsents, float):
raise ValueError('need to specify number of training set sentences, '
'not fraction, in rerun mode.')
testsettb = treebank.READERS[corpusfmt](
testcorpus.path, encoding=testcorpus.encoding,
removeempty=removeempty, morphology=morphology,
functions=functions, ensureroot=ensureroot)
if isinstance(testcorpus.numsents, float):
testcorpus.numsents = int(testcorpus.numsents
* len(testsettb.blocks()))
if testcorpus.skiptrain:
testcorpus.skip += ( # pylint: disable=maybe-no-member
traincorpus.numsents) # pylint: disable=maybe-no-member
test_blocks = OrderedDict()
test_trees = OrderedDict()
test_tagged_sents = OrderedDict()
for n, a in islice(testsettb._read_blocks(),
testcorpus.skip, testcorpus.skip # pylint: disable=maybe-no-member
+ testcorpus.numsents):
tree, sent = testsettb._parsetree(a)
if 1 <= len(sent) <= testcorpus.maxwords:
test_blocks[n] = testsettb._strblock(n, a)
test_trees[n] = tree
test_tagged_sents[n] = [(word, tag) for word, (_, tag)
in zip(sent, sorted(tree.pos()))]
logging.info('%d test sentences after length restriction <= %d',
len(test_trees), testcorpus.maxwords)
lexmodel = None
simplelexsmooth = False
test_tagged_sents_mangled = test_tagged_sents
if postagging and postagging.method in ('treetagger', 'stanford', 'frog'):
if postagging.method == 'treetagger':
# these two tags are never given by tree-tagger,
# so collect words whose tag needs to be overriden
overridetags = ('PTKANT', 'PIDAT')
elif postagging.method == 'stanford':
overridetags = ('PTKANT', )
elif postagging.method == 'frog':
overridetags = ()
taglex = defaultdict(set)
for sent in train_tagged_sents:
for word, tag in sent:
taglex[word].add(tag)
overridetagdict = {tag:
{word for word, tags in taglex.items() if tags == {tag}}
for tag in overridetags}
tagmap = {'$(': '$[', 'PAV': 'PROAV'}
test_tagged_sents_mangled = lexicon.externaltagging(postagging.method,
postagging.model, test_tagged_sents, overridetagdict, tagmap)
if postagging.retag and not rerun:
logging.info('re-tagging training corpus')
sents_to_tag = OrderedDict(enumerate(train_tagged_sents))
train_tagged_sents = lexicon.externaltagging(postagging.method,
postagging.model, sents_to_tag, overridetagdict,
tagmap).values()
for tree, tagged in zip(trees, train_tagged_sents):
for node in tree.subtrees(
lambda n: len(n) == 1 and isinstance(n[0], int)):
node.label = tagged[node[0]][1]
usetags = True # give these tags to parser
elif postagging and postagging.method == 'unknownword':
if not rerun:
sents, lexmodel = getposmodel(postagging, train_tagged_sents)
simplelexsmooth = postagging.simplelexsmooth
usetags = False # make sure gold POS tags are not given to parser
else:
usetags = True # give gold POS tags to parser
# 0: test sentences as they should be handed to the parser,
# 1: gold trees for evaluation purposes
# 2: gold sents because test sentences may be mangled by unknown word model
# 3: blocks from treebank file to reproduce the relevant part of the
# original treebank verbatim.
testset = OrderedDict((n, (
test_tagged_sents_mangled[n],
test_trees[n],
test_tagged_sents[n],
block))
for n, block in test_blocks.items())
if not test_tagged_sents:
raise ValueError('test corpus (selection) should be non-empty.')
if rerun:
trees, sents = [], []
roots = {t.label for t in trees} | {test_trees[n].label for n in testset}
if len(roots) != 1:
raise ValueError('expected unique ROOT label: %r' % roots)
top = roots.pop()
if rerun:
parser.readgrammars(resultdir, stages, postagging, top)
else:
logging.info('read training & test corpus')
getgrammars(dobinarization(trees, sents, binarization,
relationalrealizational),
sents, stages, testcorpus.maxwords, resultdir, numproc,
lexmodel, simplelexsmooth, top)
evalparam = evalmod.readparam(evalparam)
evalparam['DEBUG'] = -1
evalparam['CUTOFF_LEN'] = 40
deletelabel = evalparam.get('DELETE_LABEL', ())
deleteword = evalparam.get('DELETE_WORD', ())
begin = time.clock()
theparser = parser.Parser(stages, transformations=transformations,
binarization=binarization, postagging=postagging if postagging
and postagging.method == 'unknownword' else None,
relationalrealizational=relationalrealizational,
verbosity=verbosity)
results = doparsing(parser=theparser, testset=testset, resultdir=resultdir,
usetags=usetags, numproc=numproc, deletelabel=deletelabel,
deleteword=deleteword, corpusfmt=corpusfmt, morphology=morphology,
evalparam=evalparam)
if numproc == 1:
logging.info('time elapsed during parsing: %gs', time.clock() - begin)
for result in results:
nsent = len(result.parsetrees)
overcutoff = any(len(a) > evalparam['CUTOFF_LEN']
for a in test_tagged_sents.values())
header = (' ' + result.name.upper() + ' ').center(
44 if overcutoff else 35, '=')
evalsummary = result.evaluator.summary()
coverage = 'coverage: %s = %6.2f' % (
('%d / %d' % (nsent - result.noparse, nsent)).rjust(
25 if overcutoff else 14),
100.0 * (nsent - result.noparse) / nsent)
logging.info('\n'.join(('', header, evalsummary, coverage)))
return top