def _record_parse(self, key=None, **kwargs):
"""
Returns an element tree structure corresponding to a toolbox data file with
all markers at the same level.
Thus the following Toolbox database::
\_sh v3.0 400 Rotokas Dictionary
\_DateStampHasFourDigitYear
\lx kaa
\ps V.A
\ge gag
\gp nek i pas
\lx kaa
\ps V.B
\ge strangle
\gp pasim nek
after parsing will end up with the same structure (ignoring the extra
whitespace) as the following XML fragment after being parsed by
ElementTree::
<toolbox_data>
<header>
<_sh>v3.0 400 Rotokas Dictionary</_sh>
<_DateStampHasFourDigitYear/>
</header>
<record>
<lx>kaa</lx>
<ps>V.A</ps>
<ge>gag</ge>
<gp>nek i pas</gp>
</record>
<record>
<lx>kaa</lx>
<ps>V.B</ps>
<ge>strangle</ge>
<gp>pasim nek</gp>
</record>
</toolbox_data>
@param key: Name of key marker at the start of each record. If set to
None (the default value) the first marker that doesn't begin with an
underscore is assumed to be the key.
@type key: string
@param kwargs: Keyword arguments passed to L{StandardFormat.fields()}
@type kwargs: keyword arguments dictionary
@rtype: ElementTree._ElementInterface
@return: contents of toolbox data divided into header and records
"""
builder = TreeBuilder()
builder.start('toolbox_data', {})
builder.start('header', {})
in_records = False
for mkr, value in self.fields(**kwargs):
if key is None and not in_records and mkr[0] != '_':
key = mkr
if mkr == key:
if in_records:
builder.end('record')
else:
builder.end('header')
in_records = True
builder.start('record', {})
builder.start(mkr, {})
builder.data(value)
builder.end(mkr)
if in_records:
builder.end('record')
else:
builder.end('header')
builder.end('toolbox_data')
return builder.close()
def _record_parse(self, key=None, **kwargs):
"""
Returns an element tree structure corresponding to a toolbox data file with
all markers at the same level.
Thus the following Toolbox database::
\_sh v3.0 400 Rotokas Dictionary
\_DateStampHasFourDigitYear
\lx kaa
\ps V.A
\ge gag
\gp nek i pas
\lx kaa
\ps V.B
\ge strangle
\gp pasim nek
after parsing will end up with the same structure (ignoring the extra
whitespace) as the following XML fragment after being parsed by
ElementTree::
<toolbox_data>
<header>
<_sh>v3.0 400 Rotokas Dictionary</_sh>
<_DateStampHasFourDigitYear/>
</header>
<record>
<lx>kaa</lx>
<ps>V.A</ps>
<ge>gag</ge>
<gp>nek i pas</gp>
</record>
<record>
<lx>kaa</lx>
<ps>V.B</ps>
<ge>strangle</ge>
<gp>pasim nek</gp>
</record>
</toolbox_data>
@param key: Name of key marker at the start of each record. If set to
None (the default value) the first marker that doesn't begin with an
underscore is assumed to be the key.
@type key: string
@param kwargs: Keyword arguments passed to L{StandardFormat.fields()}
@type kwargs: keyword arguments dictionary
@rtype: ElementTree._ElementInterface
@return: contents of toolbox data divided into header and records
"""
builder = TreeBuilder()
builder.start("toolbox_data", {})
builder.start("header", {})
in_records = False
for mkr, value in self.fields(**kwargs):
if key is None and not in_records and mkr[0] != "_":
key = mkr
if mkr == key:
if in_records:
builder.end("record")
else:
builder.end("header")
in_records = True
builder.start("record", {})
builder.start(mkr, {})
builder.data(value)
builder.end(mkr)
if in_records:
builder.end("record")
else:
builder.end("header")
builder.end("toolbox_data")
return builder.close()
def grammar_parse(self, startsym, grammar, **kwargs):
"""
Returns an element tree structure corresponding to a toolbox data file
parsed according to the grammar.
@type startsym: string
@param startsym: Start symbol used for the grammar
@type grammar: dictionary of tuple of tuples
@param grammar: Contains the set of rewrite rules used to parse the
database. See the description below.
@param kwargs: Keyword arguments passed to L{toolbox.StandardFormat.fields()}
@type kwargs: keyword arguments dictionary
@rtype: ElementTree._ElementInterface
@return: Contents of toolbox data parsed according to rules in grammar
The rewrite rules in the grammar look similar to those usually used in
computer languages. The difference is that the ordering constraints
that are usually present are relaxed in this parser. The reason is that
toolbox databases seldom have consistent ordering of fields. Hence the
right side of each rule consists of a tuple with two parts. The
fields in the first part mark the start of nonterminal.
Each of them can occur only once and all those must
occur before any of the fields in the second part of that nonterminal.
Otherwise they are interpreted as marking the start
of another one of the same nonterminal. If there is more than one
in the first part of the tuple they do not need to all appear in a parse.
The fields in the second part of the tuple can occur in any order.
Sample grammar::
grammar = {
'toolbox': (('_sh',), ('_DateStampHasFourDigitYear', 'entry')),
'entry': (('lx',), ('hm', 'sense', 'dt')),
'sense': (('sn', 'ps'), ('pn', 'gv', 'dv',
'gn', 'gp', 'dn', 'rn',
'ge', 'de', 're',
'example', 'lexfunc')),
'example': (('rf', 'xv',), ('xn', 'xe')),
'lexfunc': (('lf',), ('lexvalue',)),
'lexvalue': (('lv',), ('ln', 'le')),
}
"""
parse_table, first = self._make_parse_table(grammar)
builder = TreeBuilder()
pstack = list()
state = startsym
first_elems = list()
pstack.append((state, first_elems))
builder.start(state, {})
field_iter = self.fields(**kwargs)
loop = True
try:
mkr, value = next(field_iter)
except StopIteration:
loop = False
while loop:
(state, first_elems) = pstack[-1]
if mkr in parse_table[state]:
next_state = parse_table[state][mkr]
if next_state == mkr:
if mkr in first[state]:
# may be start of a new nonterminal
if mkr not in first_elems:
# not a new nonterminal
first_elems.append(mkr)
add = True
else:
# a new nonterminal, second or subsequent instance
add = False
if len(pstack) > 1:
builder.end(state)
pstack.pop()
else:
raise ValueError('Line %d: syntax error, unexpected marker %s.' % (self.line_num, mkr))
else:
# start of terminal marker
add = True
if add:
if value:
builder.start(mkr, dict())
builder.data(value)
builder.end(mkr)
try:
mkr, value = next(field_iter)
except StopIteration:
loop = False
else:
# a non terminal, first instance
first_elems = list()
builder.start(next_state, dict())
pstack.append((next_state, first_elems))
else:
if len(pstack) > 1:
builder.end(state)
pstack.pop()
else:
raise ValueError('Line %d: syntax error, unexpected marker %s.' % (self.line_num, mkr))
for state, first_elems in reversed(pstack):
builder.end(state)
return builder.close()
def grammar_parse(self, startsym, grammar, **kwargs):
"""
Returns an element tree structure corresponding to a toolbox data file
parsed according to the grammar.
@type startsym: string
@param startsym: Start symbol used for the grammar
@type grammar: dictionary of tuple of tuples
@param grammar: Contains the set of rewrite rules used to parse the
database. See the description below.
@param kwargs: Keyword arguments passed to L{toolbox.StandardFormat.fields()}
@type kwargs: keyword arguments dictionary
@rtype: ElementTree._ElementInterface
@return: Contents of toolbox data parsed according to rules in grammar
The rewrite rules in the grammar look similar to those usually used in
computer languages. The difference is that the ordering constraints
that are usually present are relaxed in this parser. The reason is that
toolbox databases seldom have consistent ordering of fields. Hence the
right side of each rule consists of a tuple with two parts. The
fields in the first part mark the start of nonterminal.
Each of them can occur only once and all those must
occur before any of the fields in the second part of that nonterminal.
Otherwise they are interpreted as marking the start
of another one of the same nonterminal. If there is more than one
in the first part of the tuple they do not need to all appear in a parse.
The fields in the second part of the tuple can occur in any order.
Sample grammar::
grammar = {
'toolbox': (('_sh',), ('_DateStampHasFourDigitYear', 'entry')),
'entry': (('lx',), ('hm', 'sense', 'dt')),
'sense': (('sn', 'ps'), ('pn', 'gv', 'dv',
'gn', 'gp', 'dn', 'rn',
'ge', 'de', 're',
'example', 'lexfunc')),
'example': (('rf', 'xv',), ('xn', 'xe')),
'lexfunc': (('lf',), ('lexvalue',)),
'lexvalue': (('lv',), ('ln', 'le')),
}
"""
parse_table, first = self._make_parse_table(grammar)
builder = TreeBuilder()
pstack = list()
state = startsym
first_elems = list()
pstack.append((state, first_elems))
builder.start(state, {})
field_iter = self.fields(**kwargs)
loop = True
try:
mkr, value = field_iter.next()
except StopIteration:
loop = False
while loop:
(state, first_elems) = pstack[-1]
if mkr in parse_table[state]:
next_state = parse_table[state][mkr]
if next_state == mkr:
if mkr in first[state]:
# may be start of a new nonterminal
if mkr not in first_elems:
# not a new nonterminal
first_elems.append(mkr)
add = True
else:
# a new nonterminal, second or subsequent instance
add = False
if len(pstack) > 1:
builder.end(state)
pstack.pop()
else:
raise ValueError, "Line %d: syntax error, unexpected marker %s." % (self.line_num, mkr)
else:
# start of terminal marker
add = True
if add:
if value:
builder.start(mkr, dict())
builder.data(value)
builder.end(mkr)
try:
mkr, value = field_iter.next()
except StopIteration:
loop = False
else:
# a non terminal, first instance
first_elems = list()
builder.start(next_state, dict())
pstack.append((next_state, first_elems))
else:
if len(pstack) > 1:
builder.end(state)
pstack.pop()
else:
raise ValueError, "Line %d: syntax error, unexpected marker %s." % (self.line_num, mkr)
for state, first_elems in reversed(pstack):
builder.end(state)
return builder.close()