def get_word(s, cfg):
# Extract one word from the start of a given string.
# Returns a tuple containing
# - the expanded word, or None if there wasn't one. (Words may
# be empty, which isn't the same as being absent.)
# - the rest of the string
# Skip initial whitespace.
while len(s) > 0 and s[0] in whitespace:
s = s[1:]
# If there's nothing left in the string at all, return no word.
if len(s) == 0:
return (None, "")
# If there's a double quote, lex until the closing quote, and
# treat doubled quotes in mid-string as literal quotes. We
# expect whitespace or EOS immediately after the closing quote.
if s[0] == '"':
word = ""
sep = ""
while s[0] == '"':
out, s = internal_lex(s[1:], '"', 0, cfg)
word = word + sep + out
if s[0] != '"':
raise misc.builderr("unterminated double quote")
s = s[1:]
sep = '"'
if len(s) > 0 and not (s[0] in whitespace):
raise misc.builderr("expected whitespace after closing double quote")
return (word, s)
# Otherwise, just scan until whitespace.
return internal_lex(s, whitespace, 1, cfg)
def get_word(s, cfg):
# Extract one word from the start of a given string.
# Returns a tuple containing
# - the expanded word, or None if there wasn't one. (Words may
# be empty, which isn't the same as being absent.)
# - the rest of the string
# Skip initial whitespace.
while len(s) > 0 and s[0] in whitespace:
s = s[1:]
# If there's nothing left in the string at all, return no word.
if len(s) == 0:
return (None, "")
# If there's a double quote, lex until the closing quote, and
# treat doubled quotes in mid-string as literal quotes. We
# expect whitespace or EOS immediately after the closing quote.
if s[0] == '"':
word = ""
sep = ""
while s[0] == '"':
out, s = internal_lex(s[1:], '"', 0, cfg)
word = word + sep + out
if s[0] != '"':
raise misc.builderr("unterminated double quote")
s = s[1:]
sep = '"'
if len(s) > 0 and not (s[0] in whitespace):
raise misc.builderr(
"expected whitespace after closing double quote")
return (word, s)
# Otherwise, just scan until whitespace.
return internal_lex(s, whitespace, 1, cfg)
def internal_lex(s, terminatechars, permit_comments, cfg):
# Lex string s until a character in `terminatechars', or
# end-of-string, is encountered. Return a tuple consisting of
# - the lexed and expanded text before the terminating character
# - the remainder of the string, including the terminating
# character if any
out = ""
n = 0
while n < len(s):
c = s[n]
if permit_comments and c == "#":
break # effectively end-of-string
if c in terminatechars:
return (out, s[n:])
n = n + 1
if c == "$":
# Things beginning with a dollar sign are special.
c2 = s[n]
n = n + 1
if c2 == "$" or c2 == "#":
out = out + c2
elif c2 == "(":
# We have a $(...) construct. Recurse to parse the
# stuff inside the parentheses.
varname, s2 = internal_lex(s[n:], ")", 0, cfg)
if len(s2) == 0:
raise misc.builderr("`$(' without terminating `)'")
out = out + expand_varfunc(varname, cfg)
s = s2
n = 1 # eat the terminating paren
elif onecharvars.has_key(c2):
out = out + onecharvars[c2]
else:
raise misc.builderr("unrecognised character `%c' after `$'" %
c2)
else:
out = out + c
# If we reach here, this is the end of the string.
return (out, "")
def internal_lex(s, terminatechars, permit_comments, cfg):
# Lex string s until a character in `terminatechars', or
# end-of-string, is encountered. Return a tuple consisting of
# - the lexed and expanded text before the terminating character
# - the remainder of the string, including the terminating
# character if any
out = ""
n = 0
while n < len(s):
c = s[n]
if permit_comments and c == "#":
break # effectively end-of-string
if c in terminatechars:
return (out, s[n:])
n = n + 1
if c == "$":
# Things beginning with a dollar sign are special.
c2 = s[n]
n = n + 1
if c2 == "$" or c2 == "#":
out = out + c2
elif c2 == "(":
# We have a $(...) construct. Recurse to parse the
# stuff inside the parentheses.
varname, s2 = internal_lex(s[n:], ")", 0, cfg)
if len(s2) == 0:
raise misc.builderr("`$(' without terminating `)'")
out = out + expand_varfunc(varname, cfg)
s = s2
n = 1 # eat the terminating paren
elif onecharvars.has_key(c2):
out = out + onecharvars[c2]
else:
raise misc.builderr("unrecognised character `%c' after `$'" % c2)
else:
out = out + c
# If we reach here, this is the end of the string.
return (out, "")
def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module,
[cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2] + "/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False
and (cfg.force_git_svn == True or os.access(
details[2] + "/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr(
"--git-svn option given but no git-svn commit found"
)
git_native = True
mod = origmod
log.logmsg(
" git-svn ref exists but no git-svn commit found; treating as native git"
)
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i + 1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote([
"git", "--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"
])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr(
"git rev-parse command terminated with status %d" %
ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2] + "/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr(
"working directory `%s' is not a Subversion working copy" %
details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and not cfg.accept_complex_rev
and not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr(
"working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless"
% (details[2], newrev))
lcrevindex = (None, details[2])
% (details[2], newrev))
lcrevindex = (None, details[2])
else:
# Otherwise, we must read the config file to determine the
# right repository location.
save = lexer.save_vars()
lexer.set_multicharvar("module", module)
script.process_script(cfg.cfgfile, 1, cfg)
repostype = lexer.get_multicharvar("repostype")
svnrepos = gitrepos = None
if repostype == "svn":
svnrepos = lexer.get_multicharvar("svnrepos")
if svnrepos is None:
raise misc.builderr(
"Configuration file did not specify `svnrepos' for module `%s'"
% module)
log.logmsg(" Using SVN repository %s" % svnrepos)
git = False
elif repostype == "git":
gitrepos = lexer.get_multicharvar("gitrepos")
if gitrepos is None:
gitparent = lexer.get_multicharvar("gitparent")
if gitparent is None:
raise misc.builderr(
"Configuration file did not specify `gitparent' for module `%s'"
% module)
gitsuffix = lexer.get_multicharvar("gitsuffix", ".git")
gitrepos = gitparent + "/" + details[1] + gitsuffix
log.logmsg(" Using git repository %s" % gitrepos)
git = git_native = True
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath, w1)) !=
0) != (w == "ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" %
w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir +
struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline + 1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" %
exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" %
ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" %
(srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr(
"configuration does not specify a host for delegate type `%s'"
% hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(
misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]),
"r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/" + fname + "/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr(
"returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
def expand_varfunc(var, cfg):
global builddate
# Expand a variable or function enclosed in $(...). Takes a
# string containing the text from inside the parentheses (after
# any further expansion has been done on that); returns a
# string containing the expansion.
if var[0] == "!":
# `$(!' introduces a special function.
try:
pos = var.index(" ")
fn, val = var[1:pos], var[pos+1:]
except ValueError:
fn, val = var[1:], ""
if fn == "numeric":
# The entire function call has already been lexed, so
# don't lex it again.
log.logmsg("testing numericity of `%s'" % val)
if misc.numeric(val):
return "yes"
else:
return "no"
elif fn == "available":
log.logmsg("testing availability of variable `%s'" % val)
try:
get_multicharvar(val)
return "yes"
except VarInvalid:
return "no"
elif fn == "builddate":
if val != "":
raise misc.builderr("$(!builddate) expects no arguments")
if builddate is None:
# Invent a build date.
cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule)
builddate = time.strftime("%Y%m%d",time.localtime(time.time()))
verdata = checkout.verdata()
new = True
if os.path.exists(cachefile):
with open(cachefile, "r") as f:
last_builddate = f.readline().rstrip("\r\n")
last_verdata = f.read()
if verdata == last_verdata:
new = False
if new:
try:
os.mkdir(cfg.builddatecache, 0700)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
with open(cachefile + ".tmp", "w") as f:
f.write(builddate + "\n" + verdata)
os.rename(cachefile + ".tmp", cachefile)
log.logmsg("using new build date " + builddate)
else:
builddate = last_builddate
log.logmsg("reusing last build date " + builddate)
return builddate
else:
raise misc.builderr("unexpected string function `%s'" % var)
else:
# Just look up var in our list of variables, and return its
# value.
try:
return get_multicharvar(var, "")
except VarInvalid:
raise misc.builderr("special variable `%s' is not currently valid" % var)
def expand_varfunc(var, cfg):
global builddate
# Expand a variable or function enclosed in $(...). Takes a
# string containing the text from inside the parentheses (after
# any further expansion has been done on that); returns a
# string containing the expansion.
if var[0] == "!":
# `$(!' introduces a special function.
try:
pos = var.index(" ")
fn, val = var[1:pos], var[pos + 1:]
except ValueError:
fn, val = var[1:], ""
if fn == "numeric":
# The entire function call has already been lexed, so
# don't lex it again.
log.logmsg("testing numericity of `%s'" % val)
if misc.numeric(val):
return "yes"
else:
return "no"
elif fn == "available":
log.logmsg("testing availability of variable `%s'" % val)
try:
get_multicharvar(val)
return "yes"
except VarInvalid:
return "no"
elif fn == "builddate":
if val != "":
raise misc.builderr("$(!builddate) expects no arguments")
if builddate is None:
# Invent a build date.
cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule)
builddate = time.strftime("%Y%m%d",
time.localtime(time.time()))
verdata = checkout.verdata()
new = True
if os.path.exists(cachefile):
with open(cachefile, "r") as f:
last_builddate = f.readline().rstrip("\r\n")
last_verdata = f.read()
if verdata == last_verdata:
new = False
if new:
try:
os.mkdir(cfg.builddatecache, 0700)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
with open(cachefile + ".tmp", "w") as f:
f.write(builddate + "\n" + verdata)
os.rename(cachefile + ".tmp", cachefile)
log.logmsg("using new build date " + builddate)
else:
builddate = last_builddate
log.logmsg("reusing last build date " + builddate)
return builddate
else:
raise misc.builderr("unexpected string function `%s'" % var)
else:
# Just look up var in our list of variables, and return its
# value.
try:
return get_multicharvar(var, "")
except VarInvalid:
raise misc.builderr(
"special variable `%s' is not currently valid" % var)
def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module, [cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2]+"/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False and
(cfg.force_git_svn == True or
os.access(details[2]+"/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr("--git-svn option given but no git-svn commit found")
git_native = True
mod = origmod
log.logmsg(" git-svn ref exists but no git-svn commit found; treating as native git")
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i+1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote(["git",
"--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("git rev-parse command terminated with status %d" % ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2]+"/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr("working directory `%s' is not a Subversion working copy" % details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and
not cfg.accept_complex_rev and
not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev))
lcrevindex = (None, details[2])
not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev))
lcrevindex = (None, details[2])
else:
# Otherwise, we must read the config file to determine the
# right repository location.
save = lexer.save_vars()
lexer.set_multicharvar("module", module)
script.process_script(cfg.cfgfile, 1, cfg)
repostype = lexer.get_multicharvar("repostype")
svnrepos = gitrepos = None
if repostype == "svn":
svnrepos = lexer.get_multicharvar("svnrepos")
if svnrepos is None:
raise misc.builderr("Configuration file did not specify `svnrepos' for module `%s'" % module)
log.logmsg(" Using SVN repository %s" % svnrepos)
git = False
elif repostype == "git":
gitrepos = lexer.get_multicharvar("gitrepos")
if gitrepos is None:
gitparent = lexer.get_multicharvar("gitparent")
if gitparent is None:
raise misc.builderr("Configuration file did not specify `gitparent' for module `%s'" % module)
gitsuffix = lexer.get_multicharvar("gitsuffix", ".git")
gitrepos = gitparent + "/" + details[1] + gitsuffix
log.logmsg(" Using git repository %s" % gitrepos)
git = git_native = True
elif repostype is None:
raise misc.builderr("Configuration file did not specify `repostype' for module `%s'" % module)
else:
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath,w1))!=0) != (w=="ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" % w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir + struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline+1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" % exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" % ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" % (srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr("configuration does not specify a host for delegate type `%s'" % hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]), "r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/"+fname+"/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr("returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
