def get_blueprint_info(path, transform_str):
"""
Returns information about the blueprint at path. If transform_str
is given, blueprint will be transformed accordingly before returning.
"""
sheets = filereader.get_sheet_names(path)
newphase, transforms, ztransforms = \
transformer.parse_transform_str(transform_str)
result = ''
for sheet in sheets:
try:
(layers, details) = filereader.parse_file(path, sheet[1])
# transform the blueprint
if transforms is not None:
logmsg('transform', 'Transforming with: %s' % transform_str)
if newphase is not None:
details['build_type'] = buildconfig.get_full_build_type_name(newphase)
tran = Transformer(layers, details['start'])
tran.transform(transforms) # do the x/y transformations
details['start'] = tran.start
layers = tran.layers
logmsg('transform', 'Results of transform:')
loglines('transform', lambda: FileLayer.str_layers(layers))
layers = FileLayers_to_GridLayers(layers)
bp = Blueprint(sheet[0], layers, details)
# perform any requested z-transforms
if ztransforms is not None:
layers = bp.repeat_ztransforms(ztransforms, bp.layers,
Blueprint.repeater_layers)
bp.layers = layers
formatted = bp.get_info()
# add this sheet's info to the result string
result += '>>>> Sheet id %d\n' % sheet[1]
result += formatted + '\n'
except BlueprintError as ex:
continue # ignore blank/missing sheets
if result:
return result
else:
raise BlueprintError("No valid blueprints found in '%s'." % path)
def get_blueprint_info(path, transform_str):
"""
Returns information about the blueprint at path. If transform_str
is given, blueprint will be transformed accordingly before returning.
"""
sheets = filereader.get_sheet_names(path)
newphase, transforms, ztransforms = \
transformer.parse_transform_str(transform_str)
result = ''
for sheet in sheets:
try:
(layers, details) = filereader.parse_file(path, sheet[1])
# transform the blueprint
if transforms is not None:
logmsg('transform', 'Transforming with: %s' % transform_str)
if newphase is not None:
details[
'build_type'] = buildconfig.get_full_build_type_name(
newphase)
tran = Transformer(layers, details['start'])
tran.transform(transforms) # do the x/y transformations
details['start'] = tran.start
layers = tran.layers
logmsg('transform', 'Results of transform:')
loglines('transform', lambda: FileLayer.str_layers(layers))
layers = FileLayers_to_GridLayers(layers)
bp = Blueprint(sheet[0], layers, details)
# perform any requested z-transforms
if ztransforms is not None:
layers = bp.repeat_ztransforms(ztransforms, bp.layers,
Blueprint.repeater_layers)
bp.layers = layers
formatted = bp.get_info()
# add this sheet's info to the result string
result += '>>>> Sheet id %d\n' % sheet[1]
result += formatted + '\n'
except BlueprintError as ex:
continue # ignore blank/missing sheets
if result:
return result
else:
raise BlueprintError("No valid blueprints found in '%s'." % path)
def process_blueprint_command(command, startpos, transform_str, output_mode,
output_title, visualize):
"""
Parses a QF one-line command and converts it to the desired output.
"""
layers, details = filereader.parse_command(command)
logmsg('file', 'Parsed %s' % command)
loglines('file', lambda: FileLayer.str_layers(layers))
return convert_blueprint(layers, details, startpos, transform_str,
output_mode, output_title, visualize)
def process_blueprint_command(command, startpos, transform_str, output_mode,
output_title, visualize):
"""
Parses a QF one-line command and converts it to the desired output.
"""
layers, details = filereader.parse_command(command)
logmsg('file', 'Parsed %s' % command)
loglines('file', lambda: FileLayer.str_layers(layers))
return convert_blueprint(layers, details, startpos, transform_str,
output_mode, output_title, visualize)
def process_script(scriptfile, is_config, cfg):
# Log the start and end of script processing if this isn't the
# config file.
if not is_config:
log.logmsg("Opening script file %s" % scriptfile)
cfg.seen_module = 0
scriptfp = open(scriptfile, "r")
while 1:
line = scriptfp.readline()
if line == "": break
run_script_line(line, is_config, cfg)
scriptfp.close()
if not is_config:
log.logmsg("Closing script file %s" % scriptfile)
def process_script(scriptfile, is_config, cfg):
# Log the start and end of script processing if this isn't the
# config file.
if not is_config:
log.logmsg("Opening script file %s" % scriptfile)
cfg.seen_module = 0
scriptfp = open(scriptfile, "r")
while 1:
line = scriptfp.readline()
if line == "": break
run_script_line(line, is_config, cfg)
scriptfp.close()
if not is_config:
log.logmsg("Closing script file %s" % scriptfile)
def plan_route(grid, cursor):
"""
We assume the areas to be plotted are already loaded into grid.
Starting from cursor, we locate the nearest area we can plot,
and we plot it. Repeat until all areas are plotted.
"""
plots = []
grid.set_entire_grid_plottable(True)
logmsg('router', 'Starting state:')
loglines('router', lambda: Grid.str_area_labels(grid))
while (True):
nearest_pos = get_nearest_plottable_area_from(grid, cursor)
if nearest_pos is None:
# no more areas left to plot
break
else:
# record this plot start-coordinates in plots
plots.append(nearest_pos)
# mark the plot on the grid
cell = grid.get_cell(*nearest_pos)
area = cell.area
grid.set_area_cells(area, False)
logmsg('router', 'Plotting area starting at %s, area %s' % \
(nearest_pos, area))
loglines('router', lambda: Grid.str_plottable(grid))
# move cursor to the ending corner of the plotted area
cursor = area.opposite_corner(nearest_pos)
logmsg('router', 'Routed through all areas:')
loglines('router', lambda: Grid.str_area_labels(grid))
logmsg('router', 'Route replay sequence: %s' % \
''.join([grid.get_cell(*plot).label for plot in plots]))
logmsg('router', 'Cursor position now: %s' % str(cursor))
return grid, plots, cursor
def plan_route(grid, cursor):
"""
We assume the areas to be plotted are already loaded into grid.
Starting from cursor, we locate the nearest area we can plot,
and we plot it. Repeat until all areas are plotted.
"""
plots = []
grid.set_entire_grid_plottable(True)
logmsg('router', 'Starting state:')
loglines('router', lambda: Grid.str_area_labels(grid))
while (True):
nearest_pos = get_nearest_plottable_area_from(grid, cursor)
if nearest_pos is None:
# no more areas left to plot
break
else:
# record this plot start-coordinates in plots
plots.append(nearest_pos)
# mark the plot on the grid
cell = grid.get_cell(*nearest_pos)
area = cell.area
grid.set_area_cells(area, False)
logmsg('router', 'Plotting area starting at %s, area %s' % \
(nearest_pos, area))
loglines('router', lambda: Grid.str_plottable(grid))
# move cursor to the ending corner of the plotted area
cursor = area.opposite_corner(nearest_pos)
logmsg('router', 'Routed through all areas:')
loglines('router', lambda: Grid.str_area_labels(grid))
logmsg('router', 'Route replay sequence: %s' % \
''.join([grid.get_cell(*plot).label for plot in plots]))
logmsg('router', 'Cursor position now: %s' % str(cursor))
return grid, plots, cursor
def process_blueprint_file(path, sheetid, startpos, transform_str,
output_mode, output_title, visualize):
"""
Parses a blueprint file and converts it to desired output.
"""
# parse sheetid
if sheetid is None:
sheetid = 0
elif not re.match('^\d+$', str(sheetid)):
# TODO Fix this so it works
sheetid = filereader.get_sheet_names(path)[1]
# read in the blueprint
layers, details = filereader.parse_file(path, sheetid)
logmsg('file', 'Parsed %s' % path)
loglines('file', lambda: FileLayer.str_layers(layers))
return convert_blueprint(layers, details, startpos, transform_str,
output_mode, output_title, visualize)
def process_blueprint_file(path, sheetid, startpos, transform_str, output_mode,
output_title, visualize):
"""
Parses a blueprint file and converts it to desired output.
"""
# parse sheetid
if sheetid is None:
sheetid = 0
elif not re.match('^\d+$', str(sheetid)):
# TODO Fix this so it works
sheetid = filereader.get_sheet_names(path)[1]
# read in the blueprint
layers, details = filereader.parse_file(path, sheetid)
logmsg('file', 'Parsed %s' % path)
loglines('file', lambda: FileLayer.str_layers(layers))
return convert_blueprint(layers, details, startpos, transform_str,
output_mode, output_title, visualize)
def discover_areas(self):
"""
Repeatedly plot the largest contiguous areas possible until
there are no more areas left to plot.
"""
testarea = Area((0, 0), (self.grid.width - 1, self.grid.height - 1))
while True:
loglines('area', lambda: Grid.str_area_labels(self.grid))
logmsg('area', 'Marking largest plottable areas starting ' + \
'with label %s' % self.label)
self.label = self.mark_largest_plottable_areas(self.label)
# if every single cell is non-plottable (already plotted)..
if not self.grid.is_area_plottable(testarea, True):
logmsg('area', 'All areas discovered:')
loglines('area', lambda: Grid.str_area_labels(self.grid))
return
raise AreaPlotterError("Unable to plot all areas for unknown reason")
def discover_areas(self):
"""
Repeatedly plot the largest contiguous areas possible until
there are no more areas left to plot.
"""
testarea = Area((0, 0), (self.grid.width - 1, self.grid.height - 1))
while True:
loglines('area', lambda: Grid.str_area_labels(self.grid))
logmsg('area', 'Marking largest plottable areas starting ' + \
'with label %s' % self.label)
self.label = self.mark_largest_plottable_areas(self.label)
# if every single cell is non-plottable (already plotted)..
if not self.grid.is_area_plottable(testarea, True):
logmsg('area', 'All areas discovered:')
loglines('area', lambda: Grid.str_area_labels(self.grid))
return
raise AreaPlotterError("Unable to plot all areas for unknown reason")
def transform(self, transforms):
"""Transforms start, layers using the given transforms."""
layers = self.layers
start = self.start
# loop through all single-layer transformations to all layers
for i, layer in enumerate(layers):
a = layer.rows # aka the memory bucket
b = layer.rows # aka the current bucket
logmsg('transform', 'Transformation buckets before layer %d:' % i)
loglines('transform', lambda: self.str_buckets(a, b))
left = transforms
for t in transforms:
param, cmd = t
left = left[1:] # remove this cmd from the remaining cmds
if cmd == 'halign':
self.halign = param
elif cmd == 'valign':
self.valign = param
elif cmd == '!':
# The ! command just updates A to match B
a = b
else:
a, b = self.apply_transform(t, a, b) # do the transform
# adjust start pos for 'n' and 'w' commands
if cmd == 'n':
start = add_points(start, (0, layers[0].height() *
(param - 1)))
elif cmd == 'w':
start = add_points(start, (layers[0].width() *
(param - 1), 0))
if cmd in ('halign', 'valign'):
logmsg('transform', 'Set %s=%s' % (t[1], t[0]))
else:
logmsg('transform', 'Buckets after command %s%s:' % t)
loglines('transform', lambda: self.str_buckets(a, b))
# we'll return the result in b
layers[i].rows = b
self.start, self.layers = start, layers
return
def transform(self, transforms):
"""Transforms start, layers using the given transforms."""
layers = self.layers
start = self.start
# loop through all single-layer transformations to all layers
for i, layer in enumerate(layers):
a = layer.rows # aka the memory bucket
b = layer.rows # aka the current bucket
logmsg('transform', 'Transformation buckets before layer %d:' % i)
loglines('transform', lambda: self.str_buckets(a, b))
left = transforms
for t in transforms:
param, cmd = t
left = left[1:] # remove this cmd from the remaining cmds
if cmd == 'halign':
self.halign = param
elif cmd == 'valign':
self.valign = param
elif cmd == '!':
# The ! command just updates A to match B
a = b
else:
a, b = self.apply_transform(t, a, b) # do the transform
# adjust start pos for 'n' and 'w' commands
if cmd == 'n':
start = add_points(start, (0, layers[0].height() * (param - 1)))
elif cmd == 'w':
start = add_points(start, (layers[0].width() * (param - 1), 0))
if cmd in ('halign', 'valign'):
logmsg('transform', 'Set %s=%s' % (t[1], t[0]))
else:
logmsg('transform', 'Buckets after command %s%s:' % t)
loglines('transform', lambda: self.str_buckets(a, b))
# we'll return the result in b
layers[i].rows = b
self.start, self.layers = start, layers
return
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev))
lcrevindex = (None, details[2])
else:
# Otherwise, we must read the config file to determine the
# right repository location.
save = lexer.save_vars()
lexer.set_multicharvar("module", module)
script.process_script(cfg.cfgfile, 1, cfg)
repostype = lexer.get_multicharvar("repostype")
svnrepos = gitrepos = None
if repostype == "svn":
svnrepos = lexer.get_multicharvar("svnrepos")
if svnrepos is None:
raise misc.builderr("Configuration file did not specify `svnrepos' for module `%s'" % module)
log.logmsg(" Using SVN repository %s" % svnrepos)
git = False
elif repostype == "git":
gitrepos = lexer.get_multicharvar("gitrepos")
if gitrepos is None:
gitparent = lexer.get_multicharvar("gitparent")
if gitparent is None:
raise misc.builderr("Configuration file did not specify `gitparent' for module `%s'" % module)
gitsuffix = lexer.get_multicharvar("gitsuffix", ".git")
gitrepos = gitparent + "/" + details[1] + gitsuffix
log.logmsg(" Using git repository %s" % gitrepos)
git = git_native = True
elif repostype is None:
raise misc.builderr("Configuration file did not specify `repostype' for module `%s'" % module)
else:
raise misc.builderr("Configuration file specified unrecognised `repostype' for module `%s': `%s'" % (module, repostype))
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath, w1)) !=
0) != (w == "ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var, val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" %
w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir +
struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline + 1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" %
exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" %
ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" %
(srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr(
"configuration does not specify a host for delegate type `%s'"
% hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(
misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]),
"r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/" + fname + "/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr(
"returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" %
w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module, [cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2]+"/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False and
(cfg.force_git_svn == True or
os.access(details[2]+"/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr("--git-svn option given but no git-svn commit found")
git_native = True
mod = origmod
log.logmsg(" git-svn ref exists but no git-svn commit found; treating as native git")
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i+1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote(["git",
"--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("git rev-parse command terminated with status %d" % ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2]+"/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr("working directory `%s' is not a Subversion working copy" % details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and
not cfg.accept_complex_rev and
not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev))
lcrevindex = (None, details[2])
def convert_blueprint(layers, details, startpos, transform_str, output_mode,
output_title, visualize):
"""
Transforms the provided layers if required by transform_str, then renders
keystrokes/macros required to plot or visualize the blueprint specified
by layers and details and pursuant to args.
"""
# apply aliases.txt to blueprint contents
# TODO abstract this better
alii = aliases.load_aliases(
os.path.join(exetest.get_main_dir(), 'config/aliases.txt'))
layers = aliases.apply_aliases(layers, alii)
# transform the blueprint
ztransforms = []
if transform_str:
logmsg('transform', 'Transforming with: %s' % transform_str)
newphase, transforms, ztransforms = \
transformer.parse_transform_str(transform_str)
if newphase is not None:
details['build_type'] = buildconfig.get_full_build_type_name(
newphase)
tran = Transformer(layers, details['start'])
tran.transform(transforms) # do the x/y transformations
details['start'] = tran.start
layers = tran.layers
logmsg('file', 'Results of transform:')
loglines('file', lambda: FileLayer.str_layers(layers))
layers = FileLayers_to_GridLayers(layers)
if not layers: # empty blueprint handling
raise BlueprintError("Blueprint appears to be empty.")
# override starting position if startpos command line option was given
if startpos is not None:
details['start'] = parse_startpos(startpos, layers[0].grid.width,
layers[0].grid.height)
# convert layers and other data to Blueprint
bp = Blueprint('', layers, details)
# get keys/macrocode to outline or plot the blueprint
keys = []
if output_mode == 'csv':
bp.analyze()
# perform any awaiting z-transforms
layers = bp.repeat_ztransforms(ztransforms, bp.layers,
Blueprint.repeater_layers)
bp.layers = layers
output = str(bp)
else:
if visualize:
keys = bp.trace_outline()
else:
bp.analyze()
keys = bp.plot(ztransforms)
output = keystroker.convert_keys(keys, output_mode, output_title)
loglines('summary', lambda: str_summary(bp, keys))
return output
db = db.db()
log = log.log()
db.get_raffle_status()
if pid.oktorun:
try:
while True:
time.sleep(1)
# Output
out = db.get_next_output(time.time())
if out:
sendStr = 'PRIVMSG '+ out[1] +' :'+ out[2] +'\n'
print(sendStr)
irc.sendmsg(sendStr)
log.logmsg(sendStr, False)
db.delete_output(out[0])
try:
msg = irc.getmsg()
print(msg)
# Reconnect if disconnected
if len(msg) == 0:
irc.connect()
# Prevent Timeout
if irc.ping(msg):
irc.pong(msg)
# Log mentions of us
def expand_varfunc(var, cfg):
global builddate
# Expand a variable or function enclosed in $(...). Takes a
# string containing the text from inside the parentheses (after
# any further expansion has been done on that); returns a
# string containing the expansion.
if var[0] == "!":
# `$(!' introduces a special function.
try:
pos = var.index(" ")
fn, val = var[1:pos], var[pos+1:]
except ValueError:
fn, val = var[1:], ""
if fn == "numeric":
# The entire function call has already been lexed, so
# don't lex it again.
log.logmsg("testing numericity of `%s'" % val)
if misc.numeric(val):
return "yes"
else:
return "no"
elif fn == "available":
log.logmsg("testing availability of variable `%s'" % val)
try:
get_multicharvar(val)
return "yes"
except VarInvalid:
return "no"
elif fn == "builddate":
if val != "":
raise misc.builderr("$(!builddate) expects no arguments")
if builddate is None:
# Invent a build date.
cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule)
builddate = time.strftime("%Y%m%d",time.localtime(time.time()))
verdata = checkout.verdata()
new = True
if os.path.exists(cachefile):
with open(cachefile, "r") as f:
last_builddate = f.readline().rstrip("\r\n")
last_verdata = f.read()
if verdata == last_verdata:
new = False
if new:
try:
os.mkdir(cfg.builddatecache, 0700)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
with open(cachefile + ".tmp", "w") as f:
f.write(builddate + "\n" + verdata)
os.rename(cachefile + ".tmp", cachefile)
log.logmsg("using new build date " + builddate)
else:
builddate = last_builddate
log.logmsg("reusing last build date " + builddate)
return builddate
else:
raise misc.builderr("unexpected string function `%s'" % var)
else:
# Just look up var in our list of variables, and return its
# value.
try:
return get_multicharvar(var, "")
except VarInvalid:
raise misc.builderr("special variable `%s' is not currently valid" % var)
def expand_varfunc(var, cfg):
global builddate
# Expand a variable or function enclosed in $(...). Takes a
# string containing the text from inside the parentheses (after
# any further expansion has been done on that); returns a
# string containing the expansion.
if var[0] == "!":
# `$(!' introduces a special function.
try:
pos = var.index(" ")
fn, val = var[1:pos], var[pos + 1:]
except ValueError:
fn, val = var[1:], ""
if fn == "numeric":
# The entire function call has already been lexed, so
# don't lex it again.
log.logmsg("testing numericity of `%s'" % val)
if misc.numeric(val):
return "yes"
else:
return "no"
elif fn == "available":
log.logmsg("testing availability of variable `%s'" % val)
try:
get_multicharvar(val)
return "yes"
except VarInvalid:
return "no"
elif fn == "builddate":
if val != "":
raise misc.builderr("$(!builddate) expects no arguments")
if builddate is None:
# Invent a build date.
cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule)
builddate = time.strftime("%Y%m%d",
time.localtime(time.time()))
verdata = checkout.verdata()
new = True
if os.path.exists(cachefile):
with open(cachefile, "r") as f:
last_builddate = f.readline().rstrip("\r\n")
last_verdata = f.read()
if verdata == last_verdata:
new = False
if new:
try:
os.mkdir(cfg.builddatecache, 0700)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
with open(cachefile + ".tmp", "w") as f:
f.write(builddate + "\n" + verdata)
os.rename(cachefile + ".tmp", cachefile)
log.logmsg("using new build date " + builddate)
else:
builddate = last_builddate
log.logmsg("reusing last build date " + builddate)
return builddate
else:
raise misc.builderr("unexpected string function `%s'" % var)
else:
# Just look up var in our list of variables, and return its
# value.
try:
return get_multicharvar(var, "")
except VarInvalid:
raise misc.builderr(
"special variable `%s' is not currently valid" % var)
lcrevindex = (None, details[2])
else:
# Otherwise, we must read the config file to determine the
# right repository location.
save = lexer.save_vars()
lexer.set_multicharvar("module", module)
script.process_script(cfg.cfgfile, 1, cfg)
repostype = lexer.get_multicharvar("repostype")
svnrepos = gitrepos = None
if repostype == "svn":
svnrepos = lexer.get_multicharvar("svnrepos")
if svnrepos is None:
raise misc.builderr(
"Configuration file did not specify `svnrepos' for module `%s'"
% module)
log.logmsg(" Using SVN repository %s" % svnrepos)
git = False
elif repostype == "git":
gitrepos = lexer.get_multicharvar("gitrepos")
if gitrepos is None:
gitparent = lexer.get_multicharvar("gitparent")
if gitparent is None:
raise misc.builderr(
"Configuration file did not specify `gitparent' for module `%s'"
% module)
gitsuffix = lexer.get_multicharvar("gitsuffix", ".git")
gitrepos = gitparent + "/" + details[1] + gitsuffix
log.logmsg(" Using git repository %s" % gitrepos)
git = git_native = True
elif repostype is None:
raise misc.builderr(
def convert_blueprint(layers, details, startpos, transform_str,
output_mode, output_title, visualize):
"""
Transforms the provided layers if required by transform_str, then renders
keystrokes/macros required to plot or visualize the blueprint specified
by layers and details and pursuant to args.
"""
# apply aliases.txt to blueprint contents
# TODO abstract this better
alii = aliases.load_aliases(
os.path.join(exetest.get_main_dir(), 'config/aliases.txt'))
layers = aliases.apply_aliases(layers, alii)
# transform the blueprint
ztransforms = []
if transform_str:
logmsg('transform', 'Transforming with: %s' % transform_str)
newphase, transforms, ztransforms = \
transformer.parse_transform_str(transform_str)
if newphase is not None:
details['build_type'] = buildconfig.get_full_build_type_name(newphase)
tran = Transformer(layers, details['start'])
tran.transform(transforms) # do the x/y transformations
details['start'] = tran.start
layers = tran.layers
logmsg('file', 'Results of transform:')
loglines('file', lambda: FileLayer.str_layers(layers))
layers = FileLayers_to_GridLayers(layers)
if not layers: # empty blueprint handling
raise BlueprintError("Blueprint appears to be empty.")
# override starting position if startpos command line option was given
if startpos is not None:
details['start'] = parse_startpos(startpos,
layers[0].grid.width,
layers[0].grid.height)
# convert layers and other data to Blueprint
bp = Blueprint('', layers, details)
# get keys/macrocode to outline or plot the blueprint
keys = []
if output_mode == 'csv':
bp.analyze()
# perform any awaiting z-transforms
layers = bp.repeat_ztransforms(ztransforms, bp.layers,
Blueprint.repeater_layers)
bp.layers = layers
output = str(bp)
else:
if visualize:
keys = bp.trace_outline()
else:
bp.analyze()
keys = bp.plot(ztransforms)
output = keystroker.convert_keys(keys, output_mode, output_title)
loglines('summary', lambda: str_summary(bp, keys))
return output
def checkout(cfg, module, path, is_main):
log.logmsg("Checking out module %s into path %s" % (module, path))
# First, check the command-line configuration to find out how
# we need to check out this module.
details = cfg.specialrev.get(module,
[cfg.baserev, module, None, None, None])
if details[1] is None:
details[1] = module
set_headrev = 0
git = 0
git_native = False
if details[2] != None:
# If we've been given an actual working directory, we just
# do an export of that.
svnparams = [details[2]]
log.logmsg(" Using existing working directory %s" % details[2])
# Determine the revision or commit number of the working
# directory.
if os.access(details[2] + "/.git", os.F_OK):
git = 1
gitcheckoutdir = details[2]
cdcmd = misc.shellquote(["cd", gitcheckoutdir])
gitstatcmd = misc.shellquote(["git", "status"])
f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r")
mod = "M" # assume modified unless git status reports clean
while 1:
s = f.readline()
if s == "":
break
if s[-1:] == "\n": s = s[:-1]
if s[:8] == "nothing ":
mod = ""
f.close()
if (cfg.force_git_svn != False
and (cfg.force_git_svn == True or os.access(
details[2] + "/.git/refs/remotes/git-svn", os.F_OK))):
# This looks like a git-svn checkout. Scan back
# through git log to find the nearest commit that
# identifies itself as a git-svn mirror of an svn
# upstream revision, and treat it more or less as if
# it were an svn checkout from that (with optional M
# if non-git-svn-shaped commits appear first).
gitlogcmd = misc.shellquote(["git", "log"])
origmod = mod
f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r")
first = 1
while 1:
s = f.readline()
if s == "":
if cfg.force_git_svn == True:
raise misc.builderr(
"--git-svn option given but no git-svn commit found"
)
git_native = True
mod = origmod
log.logmsg(
" git-svn ref exists but no git-svn commit found; treating as native git"
)
break
if s[-1:] == "\n": s = s[:-1]
if s[:16] == " git-svn-id: ":
ss = string.split(s)
if len(ss) > 1:
try:
i = string.rindex(ss[1], "@")
newrev = ss[1][i + 1:] + mod
break
except ValueError, e:
pass
if s[:6] == "commit":
if first:
first = 0
gitcommit = string.split(s)[1]
else:
mod = "M"
f.close()
else:
# No git-svn ref exists, so assume this is a native
# git build. (Might go wrong if it's really a git
# clone of a git-svn repo from elsewhere, but I don't
# think I mind very much about that at the moment.)
git_native = True
headcmd = misc.shellquote([
"git", "--git-dir=" + gitcheckoutdir + "/.git",
"rev-parse", "HEAD"
])
log.logmsg(" Finding head revision id: " + headcmd)
f = os.popen(headcmd + " 2>&1", "r")
gitcommit = None
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
if gitcommit is None:
gitcommit = line
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr(
"git rev-parse command terminated with status %d" %
ret)
if git_native:
log.logmsg(" Native git build from commit %s" % gitcommit)
else:
log.logmsg(" Revision faked via git-svn: %s" % newrev)
elif os.access(details[2] + "/.svn", os.F_OK):
svnvcmd = misc.shellquote(["svnversion", details[2]])
f = os.popen(svnvcmd + " 2>&1", "r")
newrev = f.read()
f.close()
while newrev[-1:] == "\r" or newrev[-1:] == "\n":
newrev = newrev[:-1]
log.logmsg(" Revision returned from svnversion: %s" % newrev)
else:
raise misc.builderr(
"working directory `%s' is not a Subversion working copy" %
details[2])
# If there's more than one revision represented here, raise
# an error unless we've been told to accept that.
if (not git_native and not cfg.accept_complex_rev
and not misc.checkstr(newrev, "0123456789M")):
raise misc.builderr(
"working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless"
% (details[2], newrev))
lcrevindex = (None, details[2])
def run_script_line(s, is_config, cfg):
global delegatefps
# Execute the script line given in s.
# Trim the newline off the end of the string, to begin with.
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
w, sr = lexer.get_word(s, cfg)
if w == None or w == "":
return # no command on this line
# Log every line executed by a non-config script.
if not is_config:
log.logscript(s)
if w == "ifeq" or w == "ifneq":
w1, sr = lexer.get_word(sr, cfg)
w2, sr = lexer.get_word(sr, cfg)
log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2))
if (w1 == w2) != (w == "ifeq"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "ifexist" or w == "ifnexist":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
w1, sr = lexer.get_word(sr, cfg)
log.logmsg("testing existence of `%s'" % w1)
if (os.path.exists(os.path.join(cfg.workpath,w1))!=0) != (w=="ifexist"):
return # condition not taken
w, sr = lexer.get_word(sr, cfg) # now read the main command
if w == "set":
# Set a variable.
var, val = lexer.get_word(sr, cfg)
val = lexer.lex_all(lexer.trim(val), cfg)
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "read":
# Set a variable by reading from a file.
var, sr = lexer.get_word(sr, cfg)
filename, sr = lexer.get_word(sr, cfg)
filename = os.path.join(cfg.workpath, filename)
if not is_config:
log.logmsg("Reading file `%s'" % (filename))
with open(filename, "r") as f:
val = f.read()
val = val.rstrip("\r\n")
if not is_config:
log.logmsg("Setting variable `%s' to value `%s'" % (var,val))
lexer.set_multicharvar(var, val)
elif w == "in" or w == "in-dest":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps != None and w != "in":
raise misc.builderr("`in-dest' command invalid during delegation" % w)
dir, sr = lexer.get_word(sr, cfg)
do, sr = lexer.get_word(sr, cfg)
if do != "do":
raise misc.builderr("expected `do' after `%s'" % w)
cmd = lexer.lex_all(lexer.trim(sr), cfg)
if delegatefps != None:
log.logmsg("Running command on delegate server: " + cmd)
# Instead of running the command locally, send it to
# the delegate host, and receive in return some output
# and an exit code.
delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir + struct.pack(">L", len(cmd)) + cmd)
delegatefps[0].flush()
# Retrieve the build command's output, line by line.
output = ""
while 1:
outlen = delegatefps[1].read(4)
if len(outlen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
outlen = struct.unpack(">L", outlen)[0]
if outlen == 0:
break
outchunk = delegatefps[1].read(outlen)
if len(outchunk) < outlen:
raise misc.builderr("unexpected EOF from delegate server")
output = output + outchunk
while 1:
newline = string.find(output, "\n")
if newline < 0:
break
line = output[:newline]
output = output[newline+1:]
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
# Log the final partial line, if any.
if len(output) > 0:
while output[-1:] == "\r" or output[-1:] == "\n":
output = output[:-1]
log.logoutput(output)
exitcode = delegatefps[1].read(4)
if len(exitcode) < 4:
raise misc.builderr("unexpected EOF from delegate server")
exitcode = struct.unpack(">l", exitcode)[0]
if exitcode > 0:
raise misc.builderr("build command terminated with status %d" % exitcode)
else:
if w == "in-dest":
dir = os.path.join(cfg.outpath, dir)
else:
dir = os.path.join(cfg.workpath, dir)
log.logmsg("Running command in directory `%s': %s" % (dir, cmd))
cmd = misc.shellquote(["cd", dir]) + " && " + cmd
f = os.popen(cmd + " 2>&1", "r")
while 1:
line = f.readline()
if line == "": break
while line[-1:] == "\r" or line[-1:] == "\n":
line = line[:-1]
log.logoutput(line)
ret = f.close()
if ret > 0:
raise misc.builderr("build command terminated with status %d" % ret)
elif w == "deliver":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
srcpath, sr = lexer.get_word(sr, cfg)
sr = lexer.trim(sr)
nfiles = 0
for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)):
save = lexer.save_vars()
lexer.set_onecharvar("@", os.path.basename(srcfile))
dstfile, sx = lexer.get_word(sr, cfg)
lexer.restore_vars(save)
dstfile = os.path.join(cfg.outpath, dstfile)
log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile))
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
shutil.copyfile(srcfile, dstfile)
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("deliver statement did not match any files")
elif w == "checkout":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
module, sr = lexer.get_word(sr, cfg)
destdir, sr = lexer.get_word(sr, cfg)
if module == None or destdir == None:
raise misc.builderr("`checkout' command expects two parameters")
destdir = os.path.join(cfg.workpath, destdir)
checkout.checkout(cfg, module, destdir, 0)
elif w == "module":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
newmodule, sr = lexer.get_word(sr, cfg)
if newmodule == None:
raise misc.builderr("`module' command expects a parameter")
srcdir = os.path.join(cfg.workpath, cfg.mainmodule)
destdir = os.path.join(cfg.workpath, newmodule)
if srcdir == destdir:
log.logmsg("main module already has correct filename")
else:
log.logmsg("renaming main module directory `%s' to `%s'" % (srcdir, destdir))
os.rename(srcdir, destdir)
cfg.mainmodule = newmodule
cfg.seen_module = 1
elif w == "delegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
hosttype, sr = lexer.get_word(sr, cfg)
if hosttype == None:
raise misc.builderr("expected a host type after `delegate'")
if delegatefps != None:
raise misc.builderr("a delegation session is already open")
# Read the config file to find out what actual host to
# connect to for the given host type.
save = lexer.save_vars()
process_script(cfg.cfgfile, 1, cfg)
host = lexer.get_multicharvar("host_" + hosttype)
sshid = lexer.get_multicharvar("id_" + hosttype)
usercmd = lexer.get_multicharvar("cmd_" + hosttype)
lexer.restore_vars(save)
if host == "":
raise misc.builderr("configuration does not specify a host for delegate type `%s'" % hosttype)
# Open a connection to the delegate host.
log.logmsg("Starting delegation to host type `%s'" % hosttype)
if usercmd != None:
delcmd = usercmd
else:
if hosttype == "-":
# Special case: a host name of "-" causes a
# self-delegation, i.e. we invoke the delegate
# server directly rather than bothering with ssh.
for pdir in sys.path:
delcmd = pdir + "/" + name.server
if os.path.exists(delcmd):
break
delcmd = None
if delcmd == None:
raise misc.builderr("unable to find delegate server")
delcmd = [delcmd]
else:
delcmd = ["ssh"]
# If the user has specified an SSH identity key, use it.
if sshid != None:
delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid]
delcmd.append(host)
delcmd.append(name.server)
delcmd = misc.shellquote(delcmd)
log.logmsg(" Running delegation command: " + delcmd)
delegatefps = popen2(delcmd)
# Wait for the announcement from the far end which says the
# delegate server is running.
while 1:
s = delegatefps[1].readline()
if s == "":
raise misc.builderr("unexpected EOF from delegate server")
while s[-1:] == "\r" or s[-1:] == "\n":
s = s[:-1]
if s == name.server_banner:
log.logmsg(" Successfully started delegate server")
break
# Send a tarball of our build work directory.
tarpipe = os.popen(misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]), "r")
data = tarpipe.read()
tarpipe.close()
delegatefps[0].write("T" + struct.pack(">L", len(data)) + data)
delegatefps[0].flush()
elif w == "return":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Copy file(s) back from the delegate host. We send our
# command character "R", then a glob pattern; we then
# repeatedly read a filename and file contents until we
# receive zero filename length.
pattern, sr = lexer.get_word(sr, cfg)
if pattern == None:
raise misc.builderr("expected a file name after `return'")
delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern)
delegatefps[0].flush()
nfiles = 0
while 1:
fnamelen = delegatefps[1].read(4)
if len(fnamelen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
fnamelen = struct.unpack(">L", fnamelen)[0]
if fnamelen == 0:
break
fname = delegatefps[1].read(fnamelen)
if len(fname) < fnamelen:
raise misc.builderr("unexpected EOF from delegate server")
datalen = delegatefps[1].read(4)
if len(datalen) < 4:
raise misc.builderr("unexpected EOF from delegate server")
datalen = struct.unpack(">L", datalen)[0]
data = delegatefps[1].read(datalen)
if len(data) < datalen:
raise misc.builderr("unexpected EOF from delegate server")
log.logmsg("Returned file `%s' from delegate server" % fname) #'
# Vet the filename for obvious gotchas.
if string.find("/"+fname+"/", "/../") >= 0 or fname[:1] == "/":
raise misc.builderr("returned file `%s' failed security check" % fname) #'
dstfile = os.path.join(cfg.workpath, fname)
dstdir = os.path.dirname(dstfile)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
outfp = open(dstfile, "wb")
outfp.write(data)
outfp.close()
nfiles = nfiles + 1
if nfiles == 0:
raise misc.builderr("return statement did not match any files")
elif w == "enddelegate":
if is_config:
raise misc.builderr("`%s' command invalid in config file" % w)
if not cfg.seen_module:
raise misc.builderr("`%s' command seen before `module' command" % w)
if delegatefps == None:
raise misc.builderr("no delegation session open")
# Close the delegate session
delegatefps[0].write("Q")
delegatefps[0].close()
delegatefps[1].close()
delegatefps = None
log.logmsg("Closed delegate session")
else:
raise misc.builderr("unrecognised statement keyword `%s'" % w)
