def get_blueprint_info(path, transform_str): """ Returns information about the blueprint at path. If transform_str is given, blueprint will be transformed accordingly before returning. """ sheets = filereader.get_sheet_names(path) newphase, transforms, ztransforms = \ transformer.parse_transform_str(transform_str) result = '' for sheet in sheets: try: (layers, details) = filereader.parse_file(path, sheet[1]) # transform the blueprint if transforms is not None: logmsg('transform', 'Transforming with: %s' % transform_str) if newphase is not None: details['build_type'] = buildconfig.get_full_build_type_name(newphase) tran = Transformer(layers, details['start']) tran.transform(transforms) # do the x/y transformations details['start'] = tran.start layers = tran.layers logmsg('transform', 'Results of transform:') loglines('transform', lambda: FileLayer.str_layers(layers)) layers = FileLayers_to_GridLayers(layers) bp = Blueprint(sheet[0], layers, details) # perform any requested z-transforms if ztransforms is not None: layers = bp.repeat_ztransforms(ztransforms, bp.layers, Blueprint.repeater_layers) bp.layers = layers formatted = bp.get_info() # add this sheet's info to the result string result += '>>>> Sheet id %d\n' % sheet[1] result += formatted + '\n' except BlueprintError as ex: continue # ignore blank/missing sheets if result: return result else: raise BlueprintError("No valid blueprints found in '%s'." % path)
def get_blueprint_info(path, transform_str): """ Returns information about the blueprint at path. If transform_str is given, blueprint will be transformed accordingly before returning. """ sheets = filereader.get_sheet_names(path) newphase, transforms, ztransforms = \ transformer.parse_transform_str(transform_str) result = '' for sheet in sheets: try: (layers, details) = filereader.parse_file(path, sheet[1]) # transform the blueprint if transforms is not None: logmsg('transform', 'Transforming with: %s' % transform_str) if newphase is not None: details[ 'build_type'] = buildconfig.get_full_build_type_name( newphase) tran = Transformer(layers, details['start']) tran.transform(transforms) # do the x/y transformations details['start'] = tran.start layers = tran.layers logmsg('transform', 'Results of transform:') loglines('transform', lambda: FileLayer.str_layers(layers)) layers = FileLayers_to_GridLayers(layers) bp = Blueprint(sheet[0], layers, details) # perform any requested z-transforms if ztransforms is not None: layers = bp.repeat_ztransforms(ztransforms, bp.layers, Blueprint.repeater_layers) bp.layers = layers formatted = bp.get_info() # add this sheet's info to the result string result += '>>>> Sheet id %d\n' % sheet[1] result += formatted + '\n' except BlueprintError as ex: continue # ignore blank/missing sheets if result: return result else: raise BlueprintError("No valid blueprints found in '%s'." % path)
def process_blueprint_command(command, startpos, transform_str, output_mode, output_title, visualize): """ Parses a QF one-line command and converts it to the desired output. """ layers, details = filereader.parse_command(command) logmsg('file', 'Parsed %s' % command) loglines('file', lambda: FileLayer.str_layers(layers)) return convert_blueprint(layers, details, startpos, transform_str, output_mode, output_title, visualize)
def process_script(scriptfile, is_config, cfg): # Log the start and end of script processing if this isn't the # config file. if not is_config: log.logmsg("Opening script file %s" % scriptfile) cfg.seen_module = 0 scriptfp = open(scriptfile, "r") while 1: line = scriptfp.readline() if line == "": break run_script_line(line, is_config, cfg) scriptfp.close() if not is_config: log.logmsg("Closing script file %s" % scriptfile)
def plan_route(grid, cursor): """ We assume the areas to be plotted are already loaded into grid. Starting from cursor, we locate the nearest area we can plot, and we plot it. Repeat until all areas are plotted. """ plots = [] grid.set_entire_grid_plottable(True) logmsg('router', 'Starting state:') loglines('router', lambda: Grid.str_area_labels(grid)) while (True): nearest_pos = get_nearest_plottable_area_from(grid, cursor) if nearest_pos is None: # no more areas left to plot break else: # record this plot start-coordinates in plots plots.append(nearest_pos) # mark the plot on the grid cell = grid.get_cell(*nearest_pos) area = cell.area grid.set_area_cells(area, False) logmsg('router', 'Plotting area starting at %s, area %s' % \ (nearest_pos, area)) loglines('router', lambda: Grid.str_plottable(grid)) # move cursor to the ending corner of the plotted area cursor = area.opposite_corner(nearest_pos) logmsg('router', 'Routed through all areas:') loglines('router', lambda: Grid.str_area_labels(grid)) logmsg('router', 'Route replay sequence: %s' % \ ''.join([grid.get_cell(*plot).label for plot in plots])) logmsg('router', 'Cursor position now: %s' % str(cursor)) return grid, plots, cursor
def process_blueprint_file(path, sheetid, startpos, transform_str, output_mode, output_title, visualize): """ Parses a blueprint file and converts it to desired output. """ # parse sheetid if sheetid is None: sheetid = 0 elif not re.match('^\d+$', str(sheetid)): # TODO Fix this so it works sheetid = filereader.get_sheet_names(path)[1] # read in the blueprint layers, details = filereader.parse_file(path, sheetid) logmsg('file', 'Parsed %s' % path) loglines('file', lambda: FileLayer.str_layers(layers)) return convert_blueprint(layers, details, startpos, transform_str, output_mode, output_title, visualize)
def discover_areas(self): """ Repeatedly plot the largest contiguous areas possible until there are no more areas left to plot. """ testarea = Area((0, 0), (self.grid.width - 1, self.grid.height - 1)) while True: loglines('area', lambda: Grid.str_area_labels(self.grid)) logmsg('area', 'Marking largest plottable areas starting ' + \ 'with label %s' % self.label) self.label = self.mark_largest_plottable_areas(self.label) # if every single cell is non-plottable (already plotted).. if not self.grid.is_area_plottable(testarea, True): logmsg('area', 'All areas discovered:') loglines('area', lambda: Grid.str_area_labels(self.grid)) return raise AreaPlotterError("Unable to plot all areas for unknown reason")
def transform(self, transforms): """Transforms start, layers using the given transforms.""" layers = self.layers start = self.start # loop through all single-layer transformations to all layers for i, layer in enumerate(layers): a = layer.rows # aka the memory bucket b = layer.rows # aka the current bucket logmsg('transform', 'Transformation buckets before layer %d:' % i) loglines('transform', lambda: self.str_buckets(a, b)) left = transforms for t in transforms: param, cmd = t left = left[1:] # remove this cmd from the remaining cmds if cmd == 'halign': self.halign = param elif cmd == 'valign': self.valign = param elif cmd == '!': # The ! command just updates A to match B a = b else: a, b = self.apply_transform(t, a, b) # do the transform # adjust start pos for 'n' and 'w' commands if cmd == 'n': start = add_points(start, (0, layers[0].height() * (param - 1))) elif cmd == 'w': start = add_points(start, (layers[0].width() * (param - 1), 0)) if cmd in ('halign', 'valign'): logmsg('transform', 'Set %s=%s' % (t[1], t[0])) else: logmsg('transform', 'Buckets after command %s%s:' % t) loglines('transform', lambda: self.str_buckets(a, b)) # we'll return the result in b layers[i].rows = b self.start, self.layers = start, layers return
raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev)) lcrevindex = (None, details[2]) else: # Otherwise, we must read the config file to determine the # right repository location. save = lexer.save_vars() lexer.set_multicharvar("module", module) script.process_script(cfg.cfgfile, 1, cfg) repostype = lexer.get_multicharvar("repostype") svnrepos = gitrepos = None if repostype == "svn": svnrepos = lexer.get_multicharvar("svnrepos") if svnrepos is None: raise misc.builderr("Configuration file did not specify `svnrepos' for module `%s'" % module) log.logmsg(" Using SVN repository %s" % svnrepos) git = False elif repostype == "git": gitrepos = lexer.get_multicharvar("gitrepos") if gitrepos is None: gitparent = lexer.get_multicharvar("gitparent") if gitparent is None: raise misc.builderr("Configuration file did not specify `gitparent' for module `%s'" % module) gitsuffix = lexer.get_multicharvar("gitsuffix", ".git") gitrepos = gitparent + "/" + details[1] + gitsuffix log.logmsg(" Using git repository %s" % gitrepos) git = git_native = True elif repostype is None: raise misc.builderr("Configuration file did not specify `repostype' for module `%s'" % module) else: raise misc.builderr("Configuration file specified unrecognised `repostype' for module `%s': `%s'" % (module, repostype))
def run_script_line(s, is_config, cfg): global delegatefps # Execute the script line given in s. # Trim the newline off the end of the string, to begin with. while s[-1:] == "\r" or s[-1:] == "\n": s = s[:-1] w, sr = lexer.get_word(s, cfg) if w == None or w == "": return # no command on this line # Log every line executed by a non-config script. if not is_config: log.logscript(s) if w == "ifeq" or w == "ifneq": w1, sr = lexer.get_word(sr, cfg) w2, sr = lexer.get_word(sr, cfg) log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2)) if (w1 == w2) != (w == "ifeq"): return # condition not taken w, sr = lexer.get_word(sr, cfg) # now read the main command if w == "ifexist" or w == "ifnexist": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) w1, sr = lexer.get_word(sr, cfg) log.logmsg("testing existence of `%s'" % w1) if (os.path.exists(os.path.join(cfg.workpath, w1)) != 0) != (w == "ifexist"): return # condition not taken w, sr = lexer.get_word(sr, cfg) # now read the main command if w == "set": # Set a variable. var, val = lexer.get_word(sr, cfg) val = lexer.lex_all(lexer.trim(val), cfg) if not is_config: log.logmsg("Setting variable `%s' to value `%s'" % (var, val)) lexer.set_multicharvar(var, val) elif w == "read": # Set a variable by reading from a file. var, sr = lexer.get_word(sr, cfg) filename, sr = lexer.get_word(sr, cfg) filename = os.path.join(cfg.workpath, filename) if not is_config: log.logmsg("Reading file `%s'" % (filename)) with open(filename, "r") as f: val = f.read() val = val.rstrip("\r\n") if not is_config: log.logmsg("Setting variable `%s' to value `%s'" % (var, val)) lexer.set_multicharvar(var, val) elif w == "in" or w == "in-dest": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps != None and w != "in": raise misc.builderr("`in-dest' command invalid during delegation" % w) dir, sr = lexer.get_word(sr, cfg) do, sr = lexer.get_word(sr, cfg) if do != "do": raise misc.builderr("expected `do' after `%s'" % w) cmd = lexer.lex_all(lexer.trim(sr), cfg) if delegatefps != None: log.logmsg("Running command on delegate server: " + cmd) # Instead of running the command locally, send it to # the delegate host, and receive in return some output # and an exit code. delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir + struct.pack(">L", len(cmd)) + cmd) delegatefps[0].flush() # Retrieve the build command's output, line by line. output = "" while 1: outlen = delegatefps[1].read(4) if len(outlen) < 4: raise misc.builderr("unexpected EOF from delegate server") outlen = struct.unpack(">L", outlen)[0] if outlen == 0: break outchunk = delegatefps[1].read(outlen) if len(outchunk) < outlen: raise misc.builderr("unexpected EOF from delegate server") output = output + outchunk while 1: newline = string.find(output, "\n") if newline < 0: break line = output[:newline] output = output[newline + 1:] while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] log.logoutput(line) # Log the final partial line, if any. if len(output) > 0: while output[-1:] == "\r" or output[-1:] == "\n": output = output[:-1] log.logoutput(output) exitcode = delegatefps[1].read(4) if len(exitcode) < 4: raise misc.builderr("unexpected EOF from delegate server") exitcode = struct.unpack(">l", exitcode)[0] if exitcode > 0: raise misc.builderr("build command terminated with status %d" % exitcode) else: if w == "in-dest": dir = os.path.join(cfg.outpath, dir) else: dir = os.path.join(cfg.workpath, dir) log.logmsg("Running command in directory `%s': %s" % (dir, cmd)) cmd = misc.shellquote(["cd", dir]) + " && " + cmd f = os.popen(cmd + " 2>&1", "r") while 1: line = f.readline() if line == "": break while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] log.logoutput(line) ret = f.close() if ret > 0: raise misc.builderr("build command terminated with status %d" % ret) elif w == "deliver": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) srcpath, sr = lexer.get_word(sr, cfg) sr = lexer.trim(sr) nfiles = 0 for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)): save = lexer.save_vars() lexer.set_onecharvar("@", os.path.basename(srcfile)) dstfile, sx = lexer.get_word(sr, cfg) lexer.restore_vars(save) dstfile = os.path.join(cfg.outpath, dstfile) log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile)) dstdir = os.path.dirname(dstfile) if not os.path.exists(dstdir): os.makedirs(dstdir) shutil.copyfile(srcfile, dstfile) nfiles = nfiles + 1 if nfiles == 0: raise misc.builderr("deliver statement did not match any files") elif w == "checkout": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) module, sr = lexer.get_word(sr, cfg) destdir, sr = lexer.get_word(sr, cfg) if module == None or destdir == None: raise misc.builderr("`checkout' command expects two parameters") destdir = os.path.join(cfg.workpath, destdir) checkout.checkout(cfg, module, destdir, 0) elif w == "module": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) newmodule, sr = lexer.get_word(sr, cfg) if newmodule == None: raise misc.builderr("`module' command expects a parameter") srcdir = os.path.join(cfg.workpath, cfg.mainmodule) destdir = os.path.join(cfg.workpath, newmodule) if srcdir == destdir: log.logmsg("main module already has correct filename") else: log.logmsg("renaming main module directory `%s' to `%s'" % (srcdir, destdir)) os.rename(srcdir, destdir) cfg.mainmodule = newmodule cfg.seen_module = 1 elif w == "delegate": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) hosttype, sr = lexer.get_word(sr, cfg) if hosttype == None: raise misc.builderr("expected a host type after `delegate'") if delegatefps != None: raise misc.builderr("a delegation session is already open") # Read the config file to find out what actual host to # connect to for the given host type. save = lexer.save_vars() process_script(cfg.cfgfile, 1, cfg) host = lexer.get_multicharvar("host_" + hosttype) sshid = lexer.get_multicharvar("id_" + hosttype) usercmd = lexer.get_multicharvar("cmd_" + hosttype) lexer.restore_vars(save) if host == "": raise misc.builderr( "configuration does not specify a host for delegate type `%s'" % hosttype) # Open a connection to the delegate host. log.logmsg("Starting delegation to host type `%s'" % hosttype) if usercmd != None: delcmd = usercmd else: if hosttype == "-": # Special case: a host name of "-" causes a # self-delegation, i.e. we invoke the delegate # server directly rather than bothering with ssh. for pdir in sys.path: delcmd = pdir + "/" + name.server if os.path.exists(delcmd): break delcmd = None if delcmd == None: raise misc.builderr("unable to find delegate server") delcmd = [delcmd] else: delcmd = ["ssh"] # If the user has specified an SSH identity key, use it. if sshid != None: delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid] delcmd.append(host) delcmd.append(name.server) delcmd = misc.shellquote(delcmd) log.logmsg(" Running delegation command: " + delcmd) delegatefps = popen2(delcmd) # Wait for the announcement from the far end which says the # delegate server is running. while 1: s = delegatefps[1].readline() if s == "": raise misc.builderr("unexpected EOF from delegate server") while s[-1:] == "\r" or s[-1:] == "\n": s = s[:-1] if s == name.server_banner: log.logmsg(" Successfully started delegate server") break # Send a tarball of our build work directory. tarpipe = os.popen( misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]), "r") data = tarpipe.read() tarpipe.close() delegatefps[0].write("T" + struct.pack(">L", len(data)) + data) delegatefps[0].flush() elif w == "return": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps == None: raise misc.builderr("no delegation session open") # Copy file(s) back from the delegate host. We send our # command character "R", then a glob pattern; we then # repeatedly read a filename and file contents until we # receive zero filename length. pattern, sr = lexer.get_word(sr, cfg) if pattern == None: raise misc.builderr("expected a file name after `return'") delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern) delegatefps[0].flush() nfiles = 0 while 1: fnamelen = delegatefps[1].read(4) if len(fnamelen) < 4: raise misc.builderr("unexpected EOF from delegate server") fnamelen = struct.unpack(">L", fnamelen)[0] if fnamelen == 0: break fname = delegatefps[1].read(fnamelen) if len(fname) < fnamelen: raise misc.builderr("unexpected EOF from delegate server") datalen = delegatefps[1].read(4) if len(datalen) < 4: raise misc.builderr("unexpected EOF from delegate server") datalen = struct.unpack(">L", datalen)[0] data = delegatefps[1].read(datalen) if len(data) < datalen: raise misc.builderr("unexpected EOF from delegate server") log.logmsg("Returned file `%s' from delegate server" % fname) #' # Vet the filename for obvious gotchas. if string.find("/" + fname + "/", "/../") >= 0 or fname[:1] == "/": raise misc.builderr( "returned file `%s' failed security check" % fname) #' dstfile = os.path.join(cfg.workpath, fname) dstdir = os.path.dirname(dstfile) if not os.path.exists(dstdir): os.makedirs(dstdir) outfp = open(dstfile, "wb") outfp.write(data) outfp.close() nfiles = nfiles + 1 if nfiles == 0: raise misc.builderr("return statement did not match any files") elif w == "enddelegate": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps == None: raise misc.builderr("no delegation session open") # Close the delegate session delegatefps[0].write("Q") delegatefps[0].close() delegatefps[1].close() delegatefps = None log.logmsg("Closed delegate session") else: raise misc.builderr("unrecognised statement keyword `%s'" % w)
def checkout(cfg, module, path, is_main): log.logmsg("Checking out module %s into path %s" % (module, path)) # First, check the command-line configuration to find out how # we need to check out this module. details = cfg.specialrev.get(module, [cfg.baserev, module, None, None, None]) if details[1] is None: details[1] = module set_headrev = 0 git = 0 git_native = False if details[2] != None: # If we've been given an actual working directory, we just # do an export of that. svnparams = [details[2]] log.logmsg(" Using existing working directory %s" % details[2]) # Determine the revision or commit number of the working # directory. if os.access(details[2]+"/.git", os.F_OK): git = 1 gitcheckoutdir = details[2] cdcmd = misc.shellquote(["cd", gitcheckoutdir]) gitstatcmd = misc.shellquote(["git", "status"]) f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r") mod = "M" # assume modified unless git status reports clean while 1: s = f.readline() if s == "": break if s[-1:] == "\n": s = s[:-1] if s[:8] == "nothing ": mod = "" f.close() if (cfg.force_git_svn != False and (cfg.force_git_svn == True or os.access(details[2]+"/.git/refs/remotes/git-svn", os.F_OK))): # This looks like a git-svn checkout. Scan back # through git log to find the nearest commit that # identifies itself as a git-svn mirror of an svn # upstream revision, and treat it more or less as if # it were an svn checkout from that (with optional M # if non-git-svn-shaped commits appear first). gitlogcmd = misc.shellquote(["git", "log"]) origmod = mod f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r") first = 1 while 1: s = f.readline() if s == "": if cfg.force_git_svn == True: raise misc.builderr("--git-svn option given but no git-svn commit found") git_native = True mod = origmod log.logmsg(" git-svn ref exists but no git-svn commit found; treating as native git") break if s[-1:] == "\n": s = s[:-1] if s[:16] == " git-svn-id: ": ss = string.split(s) if len(ss) > 1: try: i = string.rindex(ss[1], "@") newrev = ss[1][i+1:] + mod break except ValueError, e: pass if s[:6] == "commit": if first: first = 0 gitcommit = string.split(s)[1] else: mod = "M" f.close() else: # No git-svn ref exists, so assume this is a native # git build. (Might go wrong if it's really a git # clone of a git-svn repo from elsewhere, but I don't # think I mind very much about that at the moment.) git_native = True headcmd = misc.shellquote(["git", "--git-dir=" + gitcheckoutdir + "/.git", "rev-parse", "HEAD"]) log.logmsg(" Finding head revision id: " + headcmd) f = os.popen(headcmd + " 2>&1", "r") gitcommit = None while 1: line = f.readline() if line == "": break while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] if gitcommit is None: gitcommit = line log.logoutput(line) ret = f.close() if ret > 0: raise misc.builderr("git rev-parse command terminated with status %d" % ret) if git_native: log.logmsg(" Native git build from commit %s" % gitcommit) else: log.logmsg(" Revision faked via git-svn: %s" % newrev) elif os.access(details[2]+"/.svn", os.F_OK): svnvcmd = misc.shellquote(["svnversion", details[2]]) f = os.popen(svnvcmd + " 2>&1", "r") newrev = f.read() f.close() while newrev[-1:] == "\r" or newrev[-1:] == "\n": newrev = newrev[:-1] log.logmsg(" Revision returned from svnversion: %s" % newrev) else: raise misc.builderr("working directory `%s' is not a Subversion working copy" % details[2]) # If there's more than one revision represented here, raise # an error unless we've been told to accept that. if (not git_native and not cfg.accept_complex_rev and not misc.checkstr(newrev, "0123456789M")): raise misc.builderr("working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev)) lcrevindex = (None, details[2])
def convert_blueprint(layers, details, startpos, transform_str, output_mode, output_title, visualize): """ Transforms the provided layers if required by transform_str, then renders keystrokes/macros required to plot or visualize the blueprint specified by layers and details and pursuant to args. """ # apply aliases.txt to blueprint contents # TODO abstract this better alii = aliases.load_aliases( os.path.join(exetest.get_main_dir(), 'config/aliases.txt')) layers = aliases.apply_aliases(layers, alii) # transform the blueprint ztransforms = [] if transform_str: logmsg('transform', 'Transforming with: %s' % transform_str) newphase, transforms, ztransforms = \ transformer.parse_transform_str(transform_str) if newphase is not None: details['build_type'] = buildconfig.get_full_build_type_name( newphase) tran = Transformer(layers, details['start']) tran.transform(transforms) # do the x/y transformations details['start'] = tran.start layers = tran.layers logmsg('file', 'Results of transform:') loglines('file', lambda: FileLayer.str_layers(layers)) layers = FileLayers_to_GridLayers(layers) if not layers: # empty blueprint handling raise BlueprintError("Blueprint appears to be empty.") # override starting position if startpos command line option was given if startpos is not None: details['start'] = parse_startpos(startpos, layers[0].grid.width, layers[0].grid.height) # convert layers and other data to Blueprint bp = Blueprint('', layers, details) # get keys/macrocode to outline or plot the blueprint keys = [] if output_mode == 'csv': bp.analyze() # perform any awaiting z-transforms layers = bp.repeat_ztransforms(ztransforms, bp.layers, Blueprint.repeater_layers) bp.layers = layers output = str(bp) else: if visualize: keys = bp.trace_outline() else: bp.analyze() keys = bp.plot(ztransforms) output = keystroker.convert_keys(keys, output_mode, output_title) loglines('summary', lambda: str_summary(bp, keys)) return output
db = db.db() log = log.log() db.get_raffle_status() if pid.oktorun: try: while True: time.sleep(1) # Output out = db.get_next_output(time.time()) if out: sendStr = 'PRIVMSG '+ out[1] +' :'+ out[2] +'\n' print(sendStr) irc.sendmsg(sendStr) log.logmsg(sendStr, False) db.delete_output(out[0]) try: msg = irc.getmsg() print(msg) # Reconnect if disconnected if len(msg) == 0: irc.connect() # Prevent Timeout if irc.ping(msg): irc.pong(msg) # Log mentions of us
def expand_varfunc(var, cfg): global builddate # Expand a variable or function enclosed in $(...). Takes a # string containing the text from inside the parentheses (after # any further expansion has been done on that); returns a # string containing the expansion. if var[0] == "!": # `$(!' introduces a special function. try: pos = var.index(" ") fn, val = var[1:pos], var[pos+1:] except ValueError: fn, val = var[1:], "" if fn == "numeric": # The entire function call has already been lexed, so # don't lex it again. log.logmsg("testing numericity of `%s'" % val) if misc.numeric(val): return "yes" else: return "no" elif fn == "available": log.logmsg("testing availability of variable `%s'" % val) try: get_multicharvar(val) return "yes" except VarInvalid: return "no" elif fn == "builddate": if val != "": raise misc.builderr("$(!builddate) expects no arguments") if builddate is None: # Invent a build date. cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule) builddate = time.strftime("%Y%m%d",time.localtime(time.time())) verdata = checkout.verdata() new = True if os.path.exists(cachefile): with open(cachefile, "r") as f: last_builddate = f.readline().rstrip("\r\n") last_verdata = f.read() if verdata == last_verdata: new = False if new: try: os.mkdir(cfg.builddatecache, 0700) except OSError as e: if e.errno != os.errno.EEXIST: raise with open(cachefile + ".tmp", "w") as f: f.write(builddate + "\n" + verdata) os.rename(cachefile + ".tmp", cachefile) log.logmsg("using new build date " + builddate) else: builddate = last_builddate log.logmsg("reusing last build date " + builddate) return builddate else: raise misc.builderr("unexpected string function `%s'" % var) else: # Just look up var in our list of variables, and return its # value. try: return get_multicharvar(var, "") except VarInvalid: raise misc.builderr("special variable `%s' is not currently valid" % var)
def expand_varfunc(var, cfg): global builddate # Expand a variable or function enclosed in $(...). Takes a # string containing the text from inside the parentheses (after # any further expansion has been done on that); returns a # string containing the expansion. if var[0] == "!": # `$(!' introduces a special function. try: pos = var.index(" ") fn, val = var[1:pos], var[pos + 1:] except ValueError: fn, val = var[1:], "" if fn == "numeric": # The entire function call has already been lexed, so # don't lex it again. log.logmsg("testing numericity of `%s'" % val) if misc.numeric(val): return "yes" else: return "no" elif fn == "available": log.logmsg("testing availability of variable `%s'" % val) try: get_multicharvar(val) return "yes" except VarInvalid: return "no" elif fn == "builddate": if val != "": raise misc.builderr("$(!builddate) expects no arguments") if builddate is None: # Invent a build date. cachefile = os.path.join(cfg.builddatecache, cfg.mainmodule) builddate = time.strftime("%Y%m%d", time.localtime(time.time())) verdata = checkout.verdata() new = True if os.path.exists(cachefile): with open(cachefile, "r") as f: last_builddate = f.readline().rstrip("\r\n") last_verdata = f.read() if verdata == last_verdata: new = False if new: try: os.mkdir(cfg.builddatecache, 0700) except OSError as e: if e.errno != os.errno.EEXIST: raise with open(cachefile + ".tmp", "w") as f: f.write(builddate + "\n" + verdata) os.rename(cachefile + ".tmp", cachefile) log.logmsg("using new build date " + builddate) else: builddate = last_builddate log.logmsg("reusing last build date " + builddate) return builddate else: raise misc.builderr("unexpected string function `%s'" % var) else: # Just look up var in our list of variables, and return its # value. try: return get_multicharvar(var, "") except VarInvalid: raise misc.builderr( "special variable `%s' is not currently valid" % var)
lcrevindex = (None, details[2]) else: # Otherwise, we must read the config file to determine the # right repository location. save = lexer.save_vars() lexer.set_multicharvar("module", module) script.process_script(cfg.cfgfile, 1, cfg) repostype = lexer.get_multicharvar("repostype") svnrepos = gitrepos = None if repostype == "svn": svnrepos = lexer.get_multicharvar("svnrepos") if svnrepos is None: raise misc.builderr( "Configuration file did not specify `svnrepos' for module `%s'" % module) log.logmsg(" Using SVN repository %s" % svnrepos) git = False elif repostype == "git": gitrepos = lexer.get_multicharvar("gitrepos") if gitrepos is None: gitparent = lexer.get_multicharvar("gitparent") if gitparent is None: raise misc.builderr( "Configuration file did not specify `gitparent' for module `%s'" % module) gitsuffix = lexer.get_multicharvar("gitsuffix", ".git") gitrepos = gitparent + "/" + details[1] + gitsuffix log.logmsg(" Using git repository %s" % gitrepos) git = git_native = True elif repostype is None: raise misc.builderr(
def convert_blueprint(layers, details, startpos, transform_str, output_mode, output_title, visualize): """ Transforms the provided layers if required by transform_str, then renders keystrokes/macros required to plot or visualize the blueprint specified by layers and details and pursuant to args. """ # apply aliases.txt to blueprint contents # TODO abstract this better alii = aliases.load_aliases( os.path.join(exetest.get_main_dir(), 'config/aliases.txt')) layers = aliases.apply_aliases(layers, alii) # transform the blueprint ztransforms = [] if transform_str: logmsg('transform', 'Transforming with: %s' % transform_str) newphase, transforms, ztransforms = \ transformer.parse_transform_str(transform_str) if newphase is not None: details['build_type'] = buildconfig.get_full_build_type_name(newphase) tran = Transformer(layers, details['start']) tran.transform(transforms) # do the x/y transformations details['start'] = tran.start layers = tran.layers logmsg('file', 'Results of transform:') loglines('file', lambda: FileLayer.str_layers(layers)) layers = FileLayers_to_GridLayers(layers) if not layers: # empty blueprint handling raise BlueprintError("Blueprint appears to be empty.") # override starting position if startpos command line option was given if startpos is not None: details['start'] = parse_startpos(startpos, layers[0].grid.width, layers[0].grid.height) # convert layers and other data to Blueprint bp = Blueprint('', layers, details) # get keys/macrocode to outline or plot the blueprint keys = [] if output_mode == 'csv': bp.analyze() # perform any awaiting z-transforms layers = bp.repeat_ztransforms(ztransforms, bp.layers, Blueprint.repeater_layers) bp.layers = layers output = str(bp) else: if visualize: keys = bp.trace_outline() else: bp.analyze() keys = bp.plot(ztransforms) output = keystroker.convert_keys(keys, output_mode, output_title) loglines('summary', lambda: str_summary(bp, keys)) return output
def checkout(cfg, module, path, is_main): log.logmsg("Checking out module %s into path %s" % (module, path)) # First, check the command-line configuration to find out how # we need to check out this module. details = cfg.specialrev.get(module, [cfg.baserev, module, None, None, None]) if details[1] is None: details[1] = module set_headrev = 0 git = 0 git_native = False if details[2] != None: # If we've been given an actual working directory, we just # do an export of that. svnparams = [details[2]] log.logmsg(" Using existing working directory %s" % details[2]) # Determine the revision or commit number of the working # directory. if os.access(details[2] + "/.git", os.F_OK): git = 1 gitcheckoutdir = details[2] cdcmd = misc.shellquote(["cd", gitcheckoutdir]) gitstatcmd = misc.shellquote(["git", "status"]) f = os.popen(cdcmd + "&&" + gitstatcmd + " 2>&1", "r") mod = "M" # assume modified unless git status reports clean while 1: s = f.readline() if s == "": break if s[-1:] == "\n": s = s[:-1] if s[:8] == "nothing ": mod = "" f.close() if (cfg.force_git_svn != False and (cfg.force_git_svn == True or os.access( details[2] + "/.git/refs/remotes/git-svn", os.F_OK))): # This looks like a git-svn checkout. Scan back # through git log to find the nearest commit that # identifies itself as a git-svn mirror of an svn # upstream revision, and treat it more or less as if # it were an svn checkout from that (with optional M # if non-git-svn-shaped commits appear first). gitlogcmd = misc.shellquote(["git", "log"]) origmod = mod f = os.popen(cdcmd + "&&" + gitlogcmd + " 2>&1", "r") first = 1 while 1: s = f.readline() if s == "": if cfg.force_git_svn == True: raise misc.builderr( "--git-svn option given but no git-svn commit found" ) git_native = True mod = origmod log.logmsg( " git-svn ref exists but no git-svn commit found; treating as native git" ) break if s[-1:] == "\n": s = s[:-1] if s[:16] == " git-svn-id: ": ss = string.split(s) if len(ss) > 1: try: i = string.rindex(ss[1], "@") newrev = ss[1][i + 1:] + mod break except ValueError, e: pass if s[:6] == "commit": if first: first = 0 gitcommit = string.split(s)[1] else: mod = "M" f.close() else: # No git-svn ref exists, so assume this is a native # git build. (Might go wrong if it's really a git # clone of a git-svn repo from elsewhere, but I don't # think I mind very much about that at the moment.) git_native = True headcmd = misc.shellquote([ "git", "--git-dir=" + gitcheckoutdir + "/.git", "rev-parse", "HEAD" ]) log.logmsg(" Finding head revision id: " + headcmd) f = os.popen(headcmd + " 2>&1", "r") gitcommit = None while 1: line = f.readline() if line == "": break while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] if gitcommit is None: gitcommit = line log.logoutput(line) ret = f.close() if ret > 0: raise misc.builderr( "git rev-parse command terminated with status %d" % ret) if git_native: log.logmsg(" Native git build from commit %s" % gitcommit) else: log.logmsg(" Revision faked via git-svn: %s" % newrev) elif os.access(details[2] + "/.svn", os.F_OK): svnvcmd = misc.shellquote(["svnversion", details[2]]) f = os.popen(svnvcmd + " 2>&1", "r") newrev = f.read() f.close() while newrev[-1:] == "\r" or newrev[-1:] == "\n": newrev = newrev[:-1] log.logmsg(" Revision returned from svnversion: %s" % newrev) else: raise misc.builderr( "working directory `%s' is not a Subversion working copy" % details[2]) # If there's more than one revision represented here, raise # an error unless we've been told to accept that. if (not git_native and not cfg.accept_complex_rev and not misc.checkstr(newrev, "0123456789M")): raise misc.builderr( "working directory `%s' has complex revision `%s'; use `--complexrev' to proceed regardless" % (details[2], newrev)) lcrevindex = (None, details[2])
def run_script_line(s, is_config, cfg): global delegatefps # Execute the script line given in s. # Trim the newline off the end of the string, to begin with. while s[-1:] == "\r" or s[-1:] == "\n": s = s[:-1] w, sr = lexer.get_word(s, cfg) if w == None or w == "": return # no command on this line # Log every line executed by a non-config script. if not is_config: log.logscript(s) if w == "ifeq" or w == "ifneq": w1, sr = lexer.get_word(sr, cfg) w2, sr = lexer.get_word(sr, cfg) log.logmsg("testing string equality of `%s' and `%s'" % (w1, w2)) if (w1 == w2) != (w == "ifeq"): return # condition not taken w, sr = lexer.get_word(sr, cfg) # now read the main command if w == "ifexist" or w == "ifnexist": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) w1, sr = lexer.get_word(sr, cfg) log.logmsg("testing existence of `%s'" % w1) if (os.path.exists(os.path.join(cfg.workpath,w1))!=0) != (w=="ifexist"): return # condition not taken w, sr = lexer.get_word(sr, cfg) # now read the main command if w == "set": # Set a variable. var, val = lexer.get_word(sr, cfg) val = lexer.lex_all(lexer.trim(val), cfg) if not is_config: log.logmsg("Setting variable `%s' to value `%s'" % (var,val)) lexer.set_multicharvar(var, val) elif w == "read": # Set a variable by reading from a file. var, sr = lexer.get_word(sr, cfg) filename, sr = lexer.get_word(sr, cfg) filename = os.path.join(cfg.workpath, filename) if not is_config: log.logmsg("Reading file `%s'" % (filename)) with open(filename, "r") as f: val = f.read() val = val.rstrip("\r\n") if not is_config: log.logmsg("Setting variable `%s' to value `%s'" % (var,val)) lexer.set_multicharvar(var, val) elif w == "in" or w == "in-dest": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps != None and w != "in": raise misc.builderr("`in-dest' command invalid during delegation" % w) dir, sr = lexer.get_word(sr, cfg) do, sr = lexer.get_word(sr, cfg) if do != "do": raise misc.builderr("expected `do' after `%s'" % w) cmd = lexer.lex_all(lexer.trim(sr), cfg) if delegatefps != None: log.logmsg("Running command on delegate server: " + cmd) # Instead of running the command locally, send it to # the delegate host, and receive in return some output # and an exit code. delegatefps[0].write("C" + struct.pack(">L", len(dir)) + dir + struct.pack(">L", len(cmd)) + cmd) delegatefps[0].flush() # Retrieve the build command's output, line by line. output = "" while 1: outlen = delegatefps[1].read(4) if len(outlen) < 4: raise misc.builderr("unexpected EOF from delegate server") outlen = struct.unpack(">L", outlen)[0] if outlen == 0: break outchunk = delegatefps[1].read(outlen) if len(outchunk) < outlen: raise misc.builderr("unexpected EOF from delegate server") output = output + outchunk while 1: newline = string.find(output, "\n") if newline < 0: break line = output[:newline] output = output[newline+1:] while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] log.logoutput(line) # Log the final partial line, if any. if len(output) > 0: while output[-1:] == "\r" or output[-1:] == "\n": output = output[:-1] log.logoutput(output) exitcode = delegatefps[1].read(4) if len(exitcode) < 4: raise misc.builderr("unexpected EOF from delegate server") exitcode = struct.unpack(">l", exitcode)[0] if exitcode > 0: raise misc.builderr("build command terminated with status %d" % exitcode) else: if w == "in-dest": dir = os.path.join(cfg.outpath, dir) else: dir = os.path.join(cfg.workpath, dir) log.logmsg("Running command in directory `%s': %s" % (dir, cmd)) cmd = misc.shellquote(["cd", dir]) + " && " + cmd f = os.popen(cmd + " 2>&1", "r") while 1: line = f.readline() if line == "": break while line[-1:] == "\r" or line[-1:] == "\n": line = line[:-1] log.logoutput(line) ret = f.close() if ret > 0: raise misc.builderr("build command terminated with status %d" % ret) elif w == "deliver": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) srcpath, sr = lexer.get_word(sr, cfg) sr = lexer.trim(sr) nfiles = 0 for srcfile in glob.glob(os.path.join(cfg.workpath, srcpath)): save = lexer.save_vars() lexer.set_onecharvar("@", os.path.basename(srcfile)) dstfile, sx = lexer.get_word(sr, cfg) lexer.restore_vars(save) dstfile = os.path.join(cfg.outpath, dstfile) log.logmsg("Delivering `%s' to `%s'" % (srcfile, dstfile)) dstdir = os.path.dirname(dstfile) if not os.path.exists(dstdir): os.makedirs(dstdir) shutil.copyfile(srcfile, dstfile) nfiles = nfiles + 1 if nfiles == 0: raise misc.builderr("deliver statement did not match any files") elif w == "checkout": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) module, sr = lexer.get_word(sr, cfg) destdir, sr = lexer.get_word(sr, cfg) if module == None or destdir == None: raise misc.builderr("`checkout' command expects two parameters") destdir = os.path.join(cfg.workpath, destdir) checkout.checkout(cfg, module, destdir, 0) elif w == "module": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) newmodule, sr = lexer.get_word(sr, cfg) if newmodule == None: raise misc.builderr("`module' command expects a parameter") srcdir = os.path.join(cfg.workpath, cfg.mainmodule) destdir = os.path.join(cfg.workpath, newmodule) if srcdir == destdir: log.logmsg("main module already has correct filename") else: log.logmsg("renaming main module directory `%s' to `%s'" % (srcdir, destdir)) os.rename(srcdir, destdir) cfg.mainmodule = newmodule cfg.seen_module = 1 elif w == "delegate": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) hosttype, sr = lexer.get_word(sr, cfg) if hosttype == None: raise misc.builderr("expected a host type after `delegate'") if delegatefps != None: raise misc.builderr("a delegation session is already open") # Read the config file to find out what actual host to # connect to for the given host type. save = lexer.save_vars() process_script(cfg.cfgfile, 1, cfg) host = lexer.get_multicharvar("host_" + hosttype) sshid = lexer.get_multicharvar("id_" + hosttype) usercmd = lexer.get_multicharvar("cmd_" + hosttype) lexer.restore_vars(save) if host == "": raise misc.builderr("configuration does not specify a host for delegate type `%s'" % hosttype) # Open a connection to the delegate host. log.logmsg("Starting delegation to host type `%s'" % hosttype) if usercmd != None: delcmd = usercmd else: if hosttype == "-": # Special case: a host name of "-" causes a # self-delegation, i.e. we invoke the delegate # server directly rather than bothering with ssh. for pdir in sys.path: delcmd = pdir + "/" + name.server if os.path.exists(delcmd): break delcmd = None if delcmd == None: raise misc.builderr("unable to find delegate server") delcmd = [delcmd] else: delcmd = ["ssh"] # If the user has specified an SSH identity key, use it. if sshid != None: delcmd = ["SSH_AUTH_SOCK="] + delcmd + ["-i", sshid] delcmd.append(host) delcmd.append(name.server) delcmd = misc.shellquote(delcmd) log.logmsg(" Running delegation command: " + delcmd) delegatefps = popen2(delcmd) # Wait for the announcement from the far end which says the # delegate server is running. while 1: s = delegatefps[1].readline() if s == "": raise misc.builderr("unexpected EOF from delegate server") while s[-1:] == "\r" or s[-1:] == "\n": s = s[:-1] if s == name.server_banner: log.logmsg(" Successfully started delegate server") break # Send a tarball of our build work directory. tarpipe = os.popen(misc.shellquote(["tar", "-C", cfg.workpath, "-czf", "-", "."]), "r") data = tarpipe.read() tarpipe.close() delegatefps[0].write("T" + struct.pack(">L", len(data)) + data) delegatefps[0].flush() elif w == "return": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps == None: raise misc.builderr("no delegation session open") # Copy file(s) back from the delegate host. We send our # command character "R", then a glob pattern; we then # repeatedly read a filename and file contents until we # receive zero filename length. pattern, sr = lexer.get_word(sr, cfg) if pattern == None: raise misc.builderr("expected a file name after `return'") delegatefps[0].write("R" + struct.pack(">L", len(pattern)) + pattern) delegatefps[0].flush() nfiles = 0 while 1: fnamelen = delegatefps[1].read(4) if len(fnamelen) < 4: raise misc.builderr("unexpected EOF from delegate server") fnamelen = struct.unpack(">L", fnamelen)[0] if fnamelen == 0: break fname = delegatefps[1].read(fnamelen) if len(fname) < fnamelen: raise misc.builderr("unexpected EOF from delegate server") datalen = delegatefps[1].read(4) if len(datalen) < 4: raise misc.builderr("unexpected EOF from delegate server") datalen = struct.unpack(">L", datalen)[0] data = delegatefps[1].read(datalen) if len(data) < datalen: raise misc.builderr("unexpected EOF from delegate server") log.logmsg("Returned file `%s' from delegate server" % fname) #' # Vet the filename for obvious gotchas. if string.find("/"+fname+"/", "/../") >= 0 or fname[:1] == "/": raise misc.builderr("returned file `%s' failed security check" % fname) #' dstfile = os.path.join(cfg.workpath, fname) dstdir = os.path.dirname(dstfile) if not os.path.exists(dstdir): os.makedirs(dstdir) outfp = open(dstfile, "wb") outfp.write(data) outfp.close() nfiles = nfiles + 1 if nfiles == 0: raise misc.builderr("return statement did not match any files") elif w == "enddelegate": if is_config: raise misc.builderr("`%s' command invalid in config file" % w) if not cfg.seen_module: raise misc.builderr("`%s' command seen before `module' command" % w) if delegatefps == None: raise misc.builderr("no delegation session open") # Close the delegate session delegatefps[0].write("Q") delegatefps[0].close() delegatefps[1].close() delegatefps = None log.logmsg("Closed delegate session") else: raise misc.builderr("unrecognised statement keyword `%s'" % w)