def iterdefinelines(it, startloc): """ Process the insides of a define. Most characters are included literally. Escaped newlines are treated as they would be in makefile syntax. Internal define/endef pairs are ignored. """ results = [] definecount = 1 for d in it: m = _redefines.match(d.s, d.lstart, d.lend) if m is not None: directive = m.group(0).strip() if directive == 'endef': definecount -= 1 if definecount == 0: return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results)) else: definecount += 1 results.append(d.s[d.lstart:d.lend]) # Falling off the end is an unterminated define! raise errors.SyntaxError("define without matching endef", startloc)
def _ensureend(d, offset, msg): """ Ensure that only whitespace remains in this data. """ s = flattenmakesyntax(d, offset) if s != '' and not s.isspace(): raise errors.SyntaxError(msg, d.getloc(offset))
def addcondition(self, loc, condition): assert isinstance(condition, Condition) condition.loc = loc if len(self._groups) and isinstance(self._groups[-1][0], ElseCondition): raise errors.SyntaxError( "Multiple else conditions for block starting at %s" % self.loc, loc) self._groups.append((condition, StatementList()))
def ifeq(d, offset): if offset > d.lend - 1: raise errors.SyntaxError("No arguments after conditional", d.getloc(offset)) # the variety of formats for this directive is rather maddening token = d.s[offset] if token not in _eqargstokenlist: raise errors.SyntaxError("No arguments after conditional", d.getloc(offset)) offset += 1 if token == '(': arg1, t, offset = parsemakesyntax(d, offset, (',', ), itermakefilechars) if t is None: raise errors.SyntaxError("Expected two arguments in conditional", d.getloc(d.lend)) arg1.rstrip() offset = d.skipwhitespace(offset) arg2, t, offset = parsemakesyntax(d, offset, (')', ), itermakefilechars) if t is None: raise errors.SyntaxError("Unexpected text in conditional", d.getloc(offset)) _ensureend(d, offset, "Unexpected text after conditional") else: arg1, t, offset = parsemakesyntax(d, offset, (token, ), itermakefilechars) if t is None: raise errors.SyntaxError("Unexpected text in conditional", d.getloc(d.lend)) offset = d.skipwhitespace(offset) if offset == d.lend: raise errors.SyntaxError("Expected two arguments in conditional", d.getloc(offset)) token = d.s[offset] if token not in '\'"': raise errors.SyntaxError("Unexpected text in conditional", d.getloc(offset)) arg2, t, offset = parsemakesyntax(d, offset + 1, (token, ), itermakefilechars) _ensureend(d, offset, "Unexpected text after conditional") return parserdata.EqCondition(arg1, arg2)
def parsemakesyntax(d, offset, stopon, iterfunc): """ Given Data, parse it into a data.Expansion. @param stopon (sequence) Indicate characters where toplevel parsing should stop. @param iterfunc (generator function) A function which is used to iterate over d, yielding (char, offset, loc) @see iterdata @see itermakefilechars @see itercommandchars @return a tuple (expansion, token, offset). If all the data is consumed, token and offset will be None """ assert callable(iterfunc) stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)), tokenlist=stopon + ('$', ), openbrace=None, closebrace=None) tokeniterator = _alltokens.finditer(d.s, offset, d.lend) di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) while True: # this is not a for loop because `di` changes during the function assert stacktop is not None try: s, token, tokenoffset, offset = next(di) except StopIteration: break stacktop.expansion.appendstr(s) if token is None: continue parsestate = stacktop.parsestate if token[0] == '$': if tokenoffset + 1 == d.lend: # an unterminated $ expands to nothing break loc = d.getloc(tokenoffset) c = token[1] if c == '$': assert len(token) == 2 stacktop.expansion.appendstr('$') elif c in ('(', '{'): closebrace = _matchingbrace[c] if len(token) > 2: fname = token[2:].rstrip() fn = functions.functionmap[fname](loc) e = data.Expansion() if len(fn) + 1 == fn.maxargs: tokenlist = (c, closebrace, '$') else: tokenlist = (',', c, closebrace, '$') stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop, e, tokenlist, function=fn, openbrace=c, closebrace=closebrace) else: e = data.Expansion() tokenlist = (':', c, closebrace, '$') stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop, e, tokenlist, openbrace=c, closebrace=closebrace, loc=loc) else: assert len(token) == 2 e = data.Expansion.fromstring(c, loc) stacktop.expansion.appendfunc(functions.VariableRef(loc, e)) elif token in ('(', '{'): assert token == stacktop.openbrace stacktop.expansion.appendstr(token) stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop, stacktop.expansion, (token, stacktop.closebrace, '$'), openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset)) elif parsestate == _PARSESTATE_PARENMATCH: assert token == stacktop.closebrace stacktop.expansion.appendstr(token) stacktop = stacktop.parent elif parsestate == _PARSESTATE_TOPLEVEL: assert stacktop.parent is None return stacktop.expansion.finish(), token, offset elif parsestate == _PARSESTATE_FUNCTION: if token == ',': stacktop.function.append(stacktop.expansion.finish()) stacktop.expansion = data.Expansion() if len(stacktop.function) + 1 == stacktop.function.maxargs: tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') stacktop.tokenlist = tokenlist elif token in (')', '}'): fn = stacktop.function fn.append(stacktop.expansion.finish()) fn.setup() stacktop = stacktop.parent stacktop.expansion.appendfunc(fn) else: assert False, "Not reached, _PARSESTATE_FUNCTION" elif parsestate == _PARSESTATE_VARNAME: if token == ':': stacktop.varname = stacktop.expansion stacktop.parsestate = _PARSESTATE_SUBSTFROM stacktop.expansion = data.Expansion() stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$') elif token in (')', '}'): fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish()) stacktop = stacktop.parent stacktop.expansion.appendfunc(fn) else: assert False, "Not reached, _PARSESTATE_VARNAME" elif parsestate == _PARSESTATE_SUBSTFROM: if token == '=': stacktop.substfrom = stacktop.expansion stacktop.parsestate = _PARSESTATE_SUBSTTO stacktop.expansion = data.Expansion() stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') elif token in (')', '}'): # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make # parses it. Issue a warning. Combine the varname and substfrom expansions to # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME _log.warning( "%s: Variable reference looks like substitution without =", stacktop.loc) stacktop.varname.appendstr(':') stacktop.varname.concat(stacktop.expansion) fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish()) stacktop = stacktop.parent stacktop.expansion.appendfunc(fn) else: assert False, "Not reached, _PARSESTATE_SUBSTFROM" elif parsestate == _PARSESTATE_SUBSTTO: assert token in (')', '}'), "Not reached, _PARSESTATE_SUBSTTO" fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(), stacktop.substfrom.finish(), stacktop.expansion.finish()) stacktop = stacktop.parent stacktop.expansion.appendfunc(fn) else: assert False, "Unexpected parse state %s" % stacktop.parsestate if stacktop.parent is not None and iterfunc == itercommandchars: di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator, ignorecomments=True) else: di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) if stacktop.parent is not None: raise errors.SyntaxError("Unterminated function call", d.getloc(offset)) assert stacktop.parsestate == _PARSESTATE_TOPLEVEL return stacktop.expansion.finish(), None, None
def parsestring(s, filename): """ Parse a string containing makefile data into a parserdata.StatementList. """ currule = False condstack = [parserdata.StatementList()] fdlines = enumeratelines(s, filename) for d in fdlines: assert len(condstack) > 0 offset = d.lstart if currule and offset < d.lend and d.s[offset] == '\t': e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars) assert token is None assert offset is None condstack[-1].append(parserdata.Command(e)) continue # To parse Makefile syntax, we first strip leading whitespace and # look for initial keywords. If there are no keywords, it's either # setting a variable or writing a rule. offset = d.skipwhitespace(offset) if offset is None: continue m = _directivesre.match(d.s, offset, d.lend) if m is not None: kword = m.group(1) offset = m.end(0) if kword == 'endif': _ensureend(d, offset, "Unexpected data after 'endif' directive") if len(condstack) == 1: raise errors.SyntaxError("unmatched 'endif' directive", d.getloc(offset)) condstack.pop().endloc = d.getloc(offset) continue if kword == 'else': if len(condstack) == 1: raise errors.SyntaxError("unmatched 'else' directive", d.getloc(offset)) m = _conditionre.match(d.s, offset, d.lend) if m is None: _ensureend(d, offset, "Unexpected data after 'else' directive.") condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition()) else: kword = m.group(1) if kword not in _conditionkeywords: raise errors.SyntaxError( "Unexpected condition after 'else' directive.", d.getloc(offset)) startoffset = offset offset = d.skipwhitespace(m.end(1)) c = _conditionkeywords[kword](d, offset) condstack[-1].addcondition(d.getloc(startoffset), c) continue if kword in _conditionkeywords: c = _conditionkeywords[kword](d, offset) cb = parserdata.ConditionBlock(d.getloc(d.lstart), c) condstack[-1].append(cb) condstack.append(cb) continue if kword == 'endef': raise errors.SyntaxError("endef without matching define", d.getloc(offset)) if kword == 'define': currule = False vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars) vname.rstrip() startloc = d.getloc(d.lstart) value = iterdefinelines(fdlines, startloc) condstack[-1].append( parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None)) continue if kword in ('include', '-include', 'includedeps', '-includedeps'): if kword.startswith('-'): required = False kword = kword[1:] else: required = True deps = kword == 'includedeps' currule = False incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) condstack[-1].append( parserdata.Include(incfile, required, deps)) continue if kword == 'vpath': currule = False e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) condstack[-1].append(parserdata.VPathDirective(e)) continue if kword == 'override': currule = False vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) vname.lstrip() vname.rstrip() if token is None: raise errors.SyntaxError( "Malformed override directive, need =", d.getloc(d.lstart)) value = flattenmakesyntax(d, offset).lstrip() condstack[-1].append( parserdata.SetVariable( vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE)) continue if kword == 'export': currule = False e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) e.lstrip() e.rstrip() if token is None: condstack[-1].append( parserdata.ExportDirective(e, concurrent_set=False)) else: condstack[-1].append( parserdata.ExportDirective(e, concurrent_set=True)) value = flattenmakesyntax(d, offset).lstrip() condstack[-1].append( parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) continue if kword == 'unexport': e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars) condstack[-1].append(parserdata.UnexportDirective(e)) continue e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars) if token is None: e.rstrip() e.lstrip() if not e.isempty(): condstack[-1].append(parserdata.EmptyDirective(e)) continue # if we encountered real makefile syntax, the current rule is over currule = False if token in _varsettokens: e.lstrip() e.rstrip() value = flattenmakesyntax(d, offset).lstrip() condstack[-1].append( parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) else: doublecolon = token == '::' # `e` is targets or target patterns, which can end up as # * a rule # * an implicit rule # * a static pattern rule # * a target-specific variable definition # * a pattern-specific variable definition # any of the rules may have order-only prerequisites # delimited by |, and a command delimited by ; targets = e e, token, offset = parsemakesyntax(d, offset, _varsettokens + (':', '|', ';'), itermakefilechars) if token in (None, ';'): condstack[-1].append(parserdata.Rule(targets, e, doublecolon)) currule = True if token == ';': offset = d.skipwhitespace(offset) e, t, offset = parsemakesyntax(d, offset, (), itercommandchars) condstack[-1].append(parserdata.Command(e)) elif token in _varsettokens: e.lstrip() e.rstrip() value = flattenmakesyntax(d, offset).lstrip() condstack[-1].append( parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets)) elif token == '|': raise errors.SyntaxError( 'order-only prerequisites not implemented', d.getloc(offset)) else: assert token == ':' # static pattern rule pattern = e deps, token, offset = parsemakesyntax(d, offset, (';', ), itermakefilechars) condstack[-1].append( parserdata.StaticPatternRule(targets, pattern, deps, doublecolon)) currule = True if token == ';': offset = d.skipwhitespace(offset) e, token, offset = parsemakesyntax(d, offset, (), itercommandchars) condstack[-1].append(parserdata.Command(e)) if len(condstack) != 1: raise errors.SyntaxError("Condition never terminated with endif", condstack[-1].loc) return condstack[0]