def iterdefinelines(it, startloc):
"""
Process the insides of a define. Most characters are included literally. Escaped newlines are treated
as they would be in makefile syntax. Internal define/endef pairs are ignored.
"""
results = []
definecount = 1
for d in it:
m = _redefines.match(d.s, d.lstart, d.lend)
if m is not None:
directive = m.group(0).strip()
if directive == 'endef':
definecount -= 1
if definecount == 0:
return _makecontinuations.sub(_replacemakecontinuations,
'\n'.join(results))
else:
definecount += 1
results.append(d.s[d.lstart:d.lend])
# Falling off the end is an unterminated define!
raise errors.SyntaxError("define without matching endef", startloc)
def _ensureend(d, offset, msg):
"""
Ensure that only whitespace remains in this data.
"""
s = flattenmakesyntax(d, offset)
if s != '' and not s.isspace():
raise errors.SyntaxError(msg, d.getloc(offset))
def addcondition(self, loc, condition):
assert isinstance(condition, Condition)
condition.loc = loc
if len(self._groups) and isinstance(self._groups[-1][0],
ElseCondition):
raise errors.SyntaxError(
"Multiple else conditions for block starting at %s" % self.loc,
loc)
self._groups.append((condition, StatementList()))
def ifeq(d, offset):
if offset > d.lend - 1:
raise errors.SyntaxError("No arguments after conditional",
d.getloc(offset))
# the variety of formats for this directive is rather maddening
token = d.s[offset]
if token not in _eqargstokenlist:
raise errors.SyntaxError("No arguments after conditional",
d.getloc(offset))
offset += 1
if token == '(':
arg1, t, offset = parsemakesyntax(d, offset, (',', ),
itermakefilechars)
if t is None:
raise errors.SyntaxError("Expected two arguments in conditional",
d.getloc(d.lend))
arg1.rstrip()
offset = d.skipwhitespace(offset)
arg2, t, offset = parsemakesyntax(d, offset, (')', ),
itermakefilechars)
if t is None:
raise errors.SyntaxError("Unexpected text in conditional",
d.getloc(offset))
_ensureend(d, offset, "Unexpected text after conditional")
else:
arg1, t, offset = parsemakesyntax(d, offset, (token, ),
itermakefilechars)
if t is None:
raise errors.SyntaxError("Unexpected text in conditional",
d.getloc(d.lend))
offset = d.skipwhitespace(offset)
if offset == d.lend:
raise errors.SyntaxError("Expected two arguments in conditional",
d.getloc(offset))
token = d.s[offset]
if token not in '\'"':
raise errors.SyntaxError("Unexpected text in conditional",
d.getloc(offset))
arg2, t, offset = parsemakesyntax(d, offset + 1, (token, ),
itermakefilechars)
_ensureend(d, offset, "Unexpected text after conditional")
return parserdata.EqCondition(arg1, arg2)
def parsemakesyntax(d, offset, stopon, iterfunc):
"""
Given Data, parse it into a data.Expansion.
@param stopon (sequence)
Indicate characters where toplevel parsing should stop.
@param iterfunc (generator function)
A function which is used to iterate over d, yielding (char, offset, loc)
@see iterdata
@see itermakefilechars
@see itercommandchars
@return a tuple (expansion, token, offset). If all the data is consumed,
token and offset will be None
"""
assert callable(iterfunc)
stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL,
None,
data.Expansion(loc=d.getloc(d.lstart)),
tokenlist=stopon + ('$', ),
openbrace=None,
closebrace=None)
tokeniterator = _alltokens.finditer(d.s, offset, d.lend)
di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
while True: # this is not a for loop because `di` changes during the function
assert stacktop is not None
try:
s, token, tokenoffset, offset = next(di)
except StopIteration:
break
stacktop.expansion.appendstr(s)
if token is None:
continue
parsestate = stacktop.parsestate
if token[0] == '$':
if tokenoffset + 1 == d.lend:
# an unterminated $ expands to nothing
break
loc = d.getloc(tokenoffset)
c = token[1]
if c == '$':
assert len(token) == 2
stacktop.expansion.appendstr('$')
elif c in ('(', '{'):
closebrace = _matchingbrace[c]
if len(token) > 2:
fname = token[2:].rstrip()
fn = functions.functionmap[fname](loc)
e = data.Expansion()
if len(fn) + 1 == fn.maxargs:
tokenlist = (c, closebrace, '$')
else:
tokenlist = (',', c, closebrace, '$')
stacktop = ParseStackFrame(_PARSESTATE_FUNCTION,
stacktop,
e,
tokenlist,
function=fn,
openbrace=c,
closebrace=closebrace)
else:
e = data.Expansion()
tokenlist = (':', c, closebrace, '$')
stacktop = ParseStackFrame(_PARSESTATE_VARNAME,
stacktop,
e,
tokenlist,
openbrace=c,
closebrace=closebrace,
loc=loc)
else:
assert len(token) == 2
e = data.Expansion.fromstring(c, loc)
stacktop.expansion.appendfunc(functions.VariableRef(loc, e))
elif token in ('(', '{'):
assert token == stacktop.openbrace
stacktop.expansion.appendstr(token)
stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH,
stacktop,
stacktop.expansion,
(token, stacktop.closebrace, '$'),
openbrace=token,
closebrace=stacktop.closebrace,
loc=d.getloc(tokenoffset))
elif parsestate == _PARSESTATE_PARENMATCH:
assert token == stacktop.closebrace
stacktop.expansion.appendstr(token)
stacktop = stacktop.parent
elif parsestate == _PARSESTATE_TOPLEVEL:
assert stacktop.parent is None
return stacktop.expansion.finish(), token, offset
elif parsestate == _PARSESTATE_FUNCTION:
if token == ',':
stacktop.function.append(stacktop.expansion.finish())
stacktop.expansion = data.Expansion()
if len(stacktop.function) + 1 == stacktop.function.maxargs:
tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
stacktop.tokenlist = tokenlist
elif token in (')', '}'):
fn = stacktop.function
fn.append(stacktop.expansion.finish())
fn.setup()
stacktop = stacktop.parent
stacktop.expansion.appendfunc(fn)
else:
assert False, "Not reached, _PARSESTATE_FUNCTION"
elif parsestate == _PARSESTATE_VARNAME:
if token == ':':
stacktop.varname = stacktop.expansion
stacktop.parsestate = _PARSESTATE_SUBSTFROM
stacktop.expansion = data.Expansion()
stacktop.tokenlist = ('=', stacktop.openbrace,
stacktop.closebrace, '$')
elif token in (')', '}'):
fn = functions.VariableRef(stacktop.loc,
stacktop.expansion.finish())
stacktop = stacktop.parent
stacktop.expansion.appendfunc(fn)
else:
assert False, "Not reached, _PARSESTATE_VARNAME"
elif parsestate == _PARSESTATE_SUBSTFROM:
if token == '=':
stacktop.substfrom = stacktop.expansion
stacktop.parsestate = _PARSESTATE_SUBSTTO
stacktop.expansion = data.Expansion()
stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace,
'$')
elif token in (')', '}'):
# A substitution of the form $(VARNAME:.ee) is probably a mistake, but make
# parses it. Issue a warning. Combine the varname and substfrom expansions to
# make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME
_log.warning(
"%s: Variable reference looks like substitution without =",
stacktop.loc)
stacktop.varname.appendstr(':')
stacktop.varname.concat(stacktop.expansion)
fn = functions.VariableRef(stacktop.loc,
stacktop.varname.finish())
stacktop = stacktop.parent
stacktop.expansion.appendfunc(fn)
else:
assert False, "Not reached, _PARSESTATE_SUBSTFROM"
elif parsestate == _PARSESTATE_SUBSTTO:
assert token in (')', '}'), "Not reached, _PARSESTATE_SUBSTTO"
fn = functions.SubstitutionRef(stacktop.loc,
stacktop.varname.finish(),
stacktop.substfrom.finish(),
stacktop.expansion.finish())
stacktop = stacktop.parent
stacktop.expansion.appendfunc(fn)
else:
assert False, "Unexpected parse state %s" % stacktop.parsestate
if stacktop.parent is not None and iterfunc == itercommandchars:
di = itermakefilechars(d,
offset,
stacktop.tokenlist,
tokeniterator,
ignorecomments=True)
else:
di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
if stacktop.parent is not None:
raise errors.SyntaxError("Unterminated function call",
d.getloc(offset))
assert stacktop.parsestate == _PARSESTATE_TOPLEVEL
return stacktop.expansion.finish(), None, None
def parsestring(s, filename):
"""
Parse a string containing makefile data into a parserdata.StatementList.
"""
currule = False
condstack = [parserdata.StatementList()]
fdlines = enumeratelines(s, filename)
for d in fdlines:
assert len(condstack) > 0
offset = d.lstart
if currule and offset < d.lend and d.s[offset] == '\t':
e, token, offset = parsemakesyntax(d, offset + 1, (),
itercommandchars)
assert token is None
assert offset is None
condstack[-1].append(parserdata.Command(e))
continue
# To parse Makefile syntax, we first strip leading whitespace and
# look for initial keywords. If there are no keywords, it's either
# setting a variable or writing a rule.
offset = d.skipwhitespace(offset)
if offset is None:
continue
m = _directivesre.match(d.s, offset, d.lend)
if m is not None:
kword = m.group(1)
offset = m.end(0)
if kword == 'endif':
_ensureend(d, offset,
"Unexpected data after 'endif' directive")
if len(condstack) == 1:
raise errors.SyntaxError("unmatched 'endif' directive",
d.getloc(offset))
condstack.pop().endloc = d.getloc(offset)
continue
if kword == 'else':
if len(condstack) == 1:
raise errors.SyntaxError("unmatched 'else' directive",
d.getloc(offset))
m = _conditionre.match(d.s, offset, d.lend)
if m is None:
_ensureend(d, offset,
"Unexpected data after 'else' directive.")
condstack[-1].addcondition(d.getloc(offset),
parserdata.ElseCondition())
else:
kword = m.group(1)
if kword not in _conditionkeywords:
raise errors.SyntaxError(
"Unexpected condition after 'else' directive.",
d.getloc(offset))
startoffset = offset
offset = d.skipwhitespace(m.end(1))
c = _conditionkeywords[kword](d, offset)
condstack[-1].addcondition(d.getloc(startoffset), c)
continue
if kword in _conditionkeywords:
c = _conditionkeywords[kword](d, offset)
cb = parserdata.ConditionBlock(d.getloc(d.lstart), c)
condstack[-1].append(cb)
condstack.append(cb)
continue
if kword == 'endef':
raise errors.SyntaxError("endef without matching define",
d.getloc(offset))
if kword == 'define':
currule = False
vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars)
vname.rstrip()
startloc = d.getloc(d.lstart)
value = iterdefinelines(fdlines, startloc)
condstack[-1].append(
parserdata.SetVariable(vname,
value=value,
valueloc=startloc,
token='=',
targetexp=None))
continue
if kword in ('include', '-include', 'includedeps', '-includedeps'):
if kword.startswith('-'):
required = False
kword = kword[1:]
else:
required = True
deps = kword == 'includedeps'
currule = False
incfile, t, offset = parsemakesyntax(d, offset, (),
itermakefilechars)
condstack[-1].append(
parserdata.Include(incfile, required, deps))
continue
if kword == 'vpath':
currule = False
e, t, offset = parsemakesyntax(d, offset, (),
itermakefilechars)
condstack[-1].append(parserdata.VPathDirective(e))
continue
if kword == 'override':
currule = False
vname, token, offset = parsemakesyntax(d, offset,
_varsettokens,
itermakefilechars)
vname.lstrip()
vname.rstrip()
if token is None:
raise errors.SyntaxError(
"Malformed override directive, need =",
d.getloc(d.lstart))
value = flattenmakesyntax(d, offset).lstrip()
condstack[-1].append(
parserdata.SetVariable(
vname,
value=value,
valueloc=d.getloc(offset),
token=token,
targetexp=None,
source=data.Variables.SOURCE_OVERRIDE))
continue
if kword == 'export':
currule = False
e, token, offset = parsemakesyntax(d, offset, _varsettokens,
itermakefilechars)
e.lstrip()
e.rstrip()
if token is None:
condstack[-1].append(
parserdata.ExportDirective(e, concurrent_set=False))
else:
condstack[-1].append(
parserdata.ExportDirective(e, concurrent_set=True))
value = flattenmakesyntax(d, offset).lstrip()
condstack[-1].append(
parserdata.SetVariable(e,
value=value,
valueloc=d.getloc(offset),
token=token,
targetexp=None))
continue
if kword == 'unexport':
e, token, offset = parsemakesyntax(d, offset, (),
itermakefilechars)
condstack[-1].append(parserdata.UnexportDirective(e))
continue
e, token, offset = parsemakesyntax(d, offset,
_varsettokens + ('::', ':'),
itermakefilechars)
if token is None:
e.rstrip()
e.lstrip()
if not e.isempty():
condstack[-1].append(parserdata.EmptyDirective(e))
continue
# if we encountered real makefile syntax, the current rule is over
currule = False
if token in _varsettokens:
e.lstrip()
e.rstrip()
value = flattenmakesyntax(d, offset).lstrip()
condstack[-1].append(
parserdata.SetVariable(e,
value=value,
valueloc=d.getloc(offset),
token=token,
targetexp=None))
else:
doublecolon = token == '::'
# `e` is targets or target patterns, which can end up as
# * a rule
# * an implicit rule
# * a static pattern rule
# * a target-specific variable definition
# * a pattern-specific variable definition
# any of the rules may have order-only prerequisites
# delimited by |, and a command delimited by ;
targets = e
e, token, offset = parsemakesyntax(d, offset,
_varsettokens + (':', '|', ';'),
itermakefilechars)
if token in (None, ';'):
condstack[-1].append(parserdata.Rule(targets, e, doublecolon))
currule = True
if token == ';':
offset = d.skipwhitespace(offset)
e, t, offset = parsemakesyntax(d, offset, (),
itercommandchars)
condstack[-1].append(parserdata.Command(e))
elif token in _varsettokens:
e.lstrip()
e.rstrip()
value = flattenmakesyntax(d, offset).lstrip()
condstack[-1].append(
parserdata.SetVariable(e,
value=value,
valueloc=d.getloc(offset),
token=token,
targetexp=targets))
elif token == '|':
raise errors.SyntaxError(
'order-only prerequisites not implemented',
d.getloc(offset))
else:
assert token == ':'
# static pattern rule
pattern = e
deps, token, offset = parsemakesyntax(d, offset, (';', ),
itermakefilechars)
condstack[-1].append(
parserdata.StaticPatternRule(targets, pattern, deps,
doublecolon))
currule = True
if token == ';':
offset = d.skipwhitespace(offset)
e, token, offset = parsemakesyntax(d, offset, (),
itercommandchars)
condstack[-1].append(parserdata.Command(e))
if len(condstack) != 1:
raise errors.SyntaxError("Condition never terminated with endif",
condstack[-1].loc)
return condstack[0]
