def save_report(self, report_dir, report_template): """save a runner results to file. """ report_dir = os.path.abspath(report_dir) # Report directory cannot already exist if os.path.exists(report_dir): bot.exit( f"{report_dir} already exists, please remove before using.") dest = copy_template(report_template, report_dir) if not dest: bot.exit(f"Error writing to {dest}.") return dest
def get_function(self, funcname): """Given a function name, return it. Exit on error if not found. """ # Import the function sys.path.insert(0, os.path.dirname(self.filename)) module = ".".join(funcname.split(".")[:-1]) funcname = funcname.split(".")[-1] try: module = import_module(module) func = getattr(module, funcname) if func is None: bot.exit(f"Cannot find {funcname}.") except: bot.exit(f"Cannot import grid function {funcname}") return func
def _savepaths_valid(self, filename, allowed=None): """For some file to be written, check that the directory exists, and a proper extension is used. """ allowed = allowed or [".json", ".pkl"] regexp = "(%s)$" % ("|".join(allowed)) dirname = os.path.dirname(filename) if not os.path.exists(dirname): bot.exit(f"{dirname} does not exist, skipping save.") elif not re.search(regexp, filename): bot.warning("%s must have extensions in %s, skipping save." % (filename, ",".join(allowed))) return False return True
def expand_args(args): """Given a grid of arguments, expand special cases into longer lists of arguments. E.g., convert an entry with these keys: into: In the case that a grid has a string identifier to point to a key in the lookup, we use that listing of values instead that should already be calculated. """ for param, settings in args.items(): # If settings is a dictionary, it has to be special case if isinstance(settings, dict): # If any settings defined not allowed, do not continue if (set(settings.keys()).difference(GRIDTEST_GRIDEXPANDERS) and param != "self"): bot.exit(f"Invalid key in grid settings {settings}") # List of values just for param values = [] # Case 1: min, max, and by if "min" in settings and "max" in settings and "by" in settings: values += custom_range(settings["min"], settings["max"], settings["by"]) elif "min" in settings and "max" in settings: values += custom_range(settings["min"], settings["max"]) # Case 2: Add a custom listing to the values elif "list" in settings: values += settings["list"] # Case 3: self refers to a previously generated object (dict allowed) elif param == "self": values = settings args[param] = values else: args[param] = settings return args
def apply_function(self, funcname, args): """Given a function (a name, or a dictionary to derive name and other options from) run some set of input variables (that are taken by the function) through it to derive a result. The result returned is used to set another variable. If a count is defined, we run the function (count) times and return a list. Otherwise, we run it once. Arguments: - funcname (str or dict) : the function name or definition - args (dict) : lookup of arguments for the function """ # Default count is 1, args == args piped into function count = 1 args = deepcopy(args or {}) # If funcname is a dictionary, derive values from it if isinstance(funcname, dict): # If there is a count, we need to multiple it by that if "count" in funcname: count = funcname["count"] # The user wants to map some defined arg to a different argument if "args" in funcname: for oldkey, newkey in funcname["args"].items(): if oldkey in args: args[newkey] = args[oldkey] # The function name is required if "func" not in funcname: bot.exit(f"{funcname} is missing func key with function name.") funcname = funcname["func"] # Get function and args that are allowed for the function func = (funcname if not isinstance(funcname, str) else self.get_function(funcname)) funcargs = intersect_args(func, args) # Run the args through the function if count == 1: return func(**funcargs) return [func(**funcargs) for c in range(count)]
def run(self, tests, cleanup=True): """run will execute a test for each entry in the list of tests. the result of the test, and error codes, are saved with the test. Arguments: - tests (gridtest.main.test.GridTest) : the GridTest object """ # Keep track of some progress for the user total = len(tests) progress = 1 # Cut out early if no tests if not tests: return results = [] to_cleanup = [] try: prefix = "[%s/%s]" % (progress, total) if self.show_progress: bot.show_progress(0, total, length=35, prefix=prefix) pool = multiprocessing.Pool(self.workers, init_worker) self.start() for name, task in tests.items(): # If a class is returned, needs to be in path too sys.path.insert(0, os.path.dirname(task.filename)) # Get the function name from the tester params = { "funcname": task.get_funcname(), "module": task.module, "filename": task.filename, "metrics": task.params.get("metrics", []), "args": task.params.get("args", {}), "returns": task.params.get("returns"), } if not self.show_progress: bot.info(f"Running test {name}") result = pool.apply_async(multi_wrapper, multi_package(test_basic, [params])) # result returns [passed, result, out, error] # Store the test with the result results.append((task, result)) while results: pair = results.pop() test, result = pair result.wait() if self.show_progress: bot.show_progress(progress, total, length=35, prefix=prefix) progress += 1 prefix = "[%s/%s]" % (progress, total) # Update the task with the result passed, result, out, err, raises = result.get() test.out = out test.err = err test.success = passed test.result = result test.raises = raises self.end() pool.close() pool.join() except (KeyboardInterrupt, SystemExit): bot.error("Keyboard interrupt detected, terminating workers!") pool.terminate() sys.exit(1) except: bot.exit("Error running task.")
def get_tests(self, regexp=None, verbose=False, cleanup=True): """get tests based on a regular expression. Arguments: - regexp (str) : if provided, only include those tests that match. """ tests = {} for parent, section in self.config.items(): for name, module in section.get("tests", {}).items(): if regexp and not re.search(regexp, name): continue # Get either the file path, module name, or relative path filename = extract_modulename(section.get("filename", ""), self.input_dir) idx = 0 # Use idx to index each test with parameters for entry in module: grid = None # Grid and args cannot both be defined if "args" in entry and "grid" in entry: bot.exit(f"{name} has defined both a grid and args.") # If we find a grid, it has to reference an existing grid if "grid" in entry and entry["grid"] not in self.grids: bot.exit( f"{name} needs grid {entry['grid']} but not found in grids." ) # If we find a grid, it has to reference an existing grid if "grid" in entry and entry["grid"] in self.grids: grid = self.grids[entry["grid"]] params = deepcopy(entry) for key in ["grid", "instance"]: if key in params: del params[key] grid.params.update(params) # A class function is tested over it's instance grid instance_grid = [{}] # If entry is defined without a grid, we need to generate it if not grid: grid = Grid(name=name, params=entry, filename=filename, refs=self.grids) # If the grid has an instance, add the correct args to it if "self" in grid.args and "grid" in grid.args["self"]: instance_grid = self.grids.get( grid.args["self"]["grid"], [{}]) # If the grid is cached, we already have parameter sets argsets = grid if grid.cache: argsets = grid.argsets # iterate over argsets for a grid, get overlapping args for extra_args in instance_grid: for argset in argsets: updated = deepcopy(grid.params) # Add instance args, if needed updated["args"] = argset if extra_args: updated["args"]["self"] = extra_args tests["%s.%s" % (name, idx)] = GridTest( module=parent, name=name, params=updated, verbose=verbose, cleanup=cleanup, filename=filename, show_progress=self.show_progress, ) print(f"generating test {idx}", end="\r") idx += 1 return tests