def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module,
[cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2] + "/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False
and (cfg.force_git_svn == True or os.access(
details[2] + "/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr(
"--git-svn option given but no git-svn commit found"
)
git_native = True
mod = origmod
log.logmsg(
" git-svn ref exists but no git-svn commit found; treating as native git"
)
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i + 1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote([
"git", "--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"
])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr(
"git rev-parse command terminated with status %d" %
ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2] + "/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr(
"working directory `%s' is not a Subversion working copy" %
details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and not cfg.accept_complex_rev
and not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr(
"working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless"
% (details[2], newrev))
lcrevindex = (None, details[2])
details[1] + ".checkout.tmp")
clonecmd = ["git", "clone"]
if details[4] is None:
if details[3] is not None:
clonecmd.extend(["-b", details[3]])
clonecmd.append("--depth=1")
clonecmd.extend(["--recursive", gitrepos, gitcheckoutdir])
clonecmd = misc.shellquote(clonecmd)
log.logmsg(" Running git clone command: " + clonecmd)
f = os.popen(clonecmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr(
"git clone command terminated with status %d" % ret)
if details[4] is not None:
checkoutcmd = (
misc.shellquote(["cd", gitcheckoutdir]) + " && " +
misc.shellquote(["git", "checkout", details[4]]))
log.logmsg(" Running git checkout command: " + checkoutcmd)
f = os.popen(checkoutcmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath, w1)) !=
0) != (w == "ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" %
w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir +
struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline + 1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" %
exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" %
ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" %
(srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr(
"configuration does not specify a host for delegate type `%s'"
% hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(
misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]),
"r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/" + fname + "/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr(
"returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module, [cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2]+"/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False and
(cfg.force_git_svn == True or
os.access(details[2]+"/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr("--git-svn option given but no git-svn commit found")
git_native = True
mod = origmod
log.logmsg(" git-svn ref exists but no git-svn commit found; treating as native git")
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i+1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote(["git",
"--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("git rev-parse command terminated with status %d" % ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2]+"/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr("working directory `%s' is not a Subversion working copy" % details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and
not cfg.accept_complex_rev and
not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev))
lcrevindex = (None, details[2])
clonecmd = ["git", "clone"]
if details[4] is None:
if details[3] is not None:
clonecmd.extend(["-b", details[3]])
clonecmd.append("--depth=1")
clonecmd.extend(["--recursive",
gitrepos, gitcheckoutdir])
clonecmd = misc.shellquote(clonecmd)
log.logmsg(" Running git clone command: " + clonecmd)
f = os.popen(clonecmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("git clone command terminated with status %d" % ret)
if details[4] is not None:
checkoutcmd = (misc.shellquote(["cd", gitcheckoutdir]) +
" && " +
misc.shellquote(["git", "checkout",
details[4]]))
log.logmsg(" Running git checkout command: " + checkoutcmd)
f = os.popen(checkoutcmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath,w1))!=0) != (w=="ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" % w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir + struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline+1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" % exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" % ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" % (srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr("configuration does not specify a host for delegate type `%s'" % hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]), "r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/"+fname+"/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr("returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
