def save_report(self, report_dir, report_template):
"""save a runner results to file.
"""
report_dir = os.path.abspath(report_dir)
# Report directory cannot already exist
if os.path.exists(report_dir):
bot.exit(
f"{report_dir} already exists, please remove before using.")
dest = copy_template(report_template, report_dir)
if not dest:
bot.exit(f"Error writing to {dest}.")
return dest
def get_function(self, funcname):
"""Given a function name, return it. Exit on error if not found.
"""
# Import the function
sys.path.insert(0, os.path.dirname(self.filename))
module = ".".join(funcname.split(".")[:-1])
funcname = funcname.split(".")[-1]
try:
module = import_module(module)
func = getattr(module, funcname)
if func is None:
bot.exit(f"Cannot find {funcname}.")
except:
bot.exit(f"Cannot import grid function {funcname}")
return func
def _savepaths_valid(self, filename, allowed=None):
"""For some file to be written, check that the directory exists, and
a proper extension is used.
"""
allowed = allowed or [".json", ".pkl"]
regexp = "(%s)$" % ("|".join(allowed))
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
bot.exit(f"{dirname} does not exist, skipping save.")
elif not re.search(regexp, filename):
bot.warning("%s must have extensions in %s, skipping save." %
(filename, ",".join(allowed)))
return False
return True
def expand_args(args):
"""Given a grid of arguments, expand special cases into longer lists
of arguments.
E.g., convert an entry with these keys:
into:
In the case that a grid has a string identifier to point to a key
in the lookup, we use that listing of values instead that should
already be calculated.
"""
for param, settings in args.items():
# If settings is a dictionary, it has to be special case
if isinstance(settings, dict):
# If any settings defined not allowed, do not continue
if (set(settings.keys()).difference(GRIDTEST_GRIDEXPANDERS)
and param != "self"):
bot.exit(f"Invalid key in grid settings {settings}")
# List of values just for param
values = []
# Case 1: min, max, and by
if "min" in settings and "max" in settings and "by" in settings:
values += custom_range(settings["min"], settings["max"],
settings["by"])
elif "min" in settings and "max" in settings:
values += custom_range(settings["min"], settings["max"])
# Case 2: Add a custom listing to the values
elif "list" in settings:
values += settings["list"]
# Case 3: self refers to a previously generated object (dict allowed)
elif param == "self":
values = settings
args[param] = values
else:
args[param] = settings
return args
def apply_function(self, funcname, args):
"""Given a function (a name, or a dictionary to derive name and other
options from) run some set of input variables (that are taken by
the function) through it to derive a result. The result returned
is used to set another variable. If a count is defined, we
run the function (count) times and return a list. Otherwise, we
run it once.
Arguments:
- funcname (str or dict) : the function name or definition
- args (dict) : lookup of arguments for the function
"""
# Default count is 1, args == args piped into function
count = 1
args = deepcopy(args or {})
# If funcname is a dictionary, derive values from it
if isinstance(funcname, dict):
# If there is a count, we need to multiple it by that
if "count" in funcname:
count = funcname["count"]
# The user wants to map some defined arg to a different argument
if "args" in funcname:
for oldkey, newkey in funcname["args"].items():
if oldkey in args:
args[newkey] = args[oldkey]
# The function name is required
if "func" not in funcname:
bot.exit(f"{funcname} is missing func key with function name.")
funcname = funcname["func"]
# Get function and args that are allowed for the function
func = (funcname if not isinstance(funcname, str) else
self.get_function(funcname))
funcargs = intersect_args(func, args)
# Run the args through the function
if count == 1:
return func(**funcargs)
return [func(**funcargs) for c in range(count)]
def run(self, tests, cleanup=True):
"""run will execute a test for each entry in the list of tests.
the result of the test, and error codes, are saved with the test.
Arguments:
- tests (gridtest.main.test.GridTest) : the GridTest object
"""
# Keep track of some progress for the user
total = len(tests)
progress = 1
# Cut out early if no tests
if not tests:
return
results = []
to_cleanup = []
try:
prefix = "[%s/%s]" % (progress, total)
if self.show_progress:
bot.show_progress(0, total, length=35, prefix=prefix)
pool = multiprocessing.Pool(self.workers, init_worker)
self.start()
for name, task in tests.items():
# If a class is returned, needs to be in path too
sys.path.insert(0, os.path.dirname(task.filename))
# Get the function name from the tester
params = {
"funcname": task.get_funcname(),
"module": task.module,
"filename": task.filename,
"metrics": task.params.get("metrics", []),
"args": task.params.get("args", {}),
"returns": task.params.get("returns"),
}
if not self.show_progress:
bot.info(f"Running test {name}")
result = pool.apply_async(multi_wrapper,
multi_package(test_basic, [params]))
# result returns [passed, result, out, error]
# Store the test with the result
results.append((task, result))
while results:
pair = results.pop()
test, result = pair
result.wait()
if self.show_progress:
bot.show_progress(progress,
total,
length=35,
prefix=prefix)
progress += 1
prefix = "[%s/%s]" % (progress, total)
# Update the task with the result
passed, result, out, err, raises = result.get()
test.out = out
test.err = err
test.success = passed
test.result = result
test.raises = raises
self.end()
pool.close()
pool.join()
except (KeyboardInterrupt, SystemExit):
bot.error("Keyboard interrupt detected, terminating workers!")
pool.terminate()
sys.exit(1)
except:
bot.exit("Error running task.")
def get_tests(self, regexp=None, verbose=False, cleanup=True):
"""get tests based on a regular expression.
Arguments:
- regexp (str) : if provided, only include those tests that match.
"""
tests = {}
for parent, section in self.config.items():
for name, module in section.get("tests", {}).items():
if regexp and not re.search(regexp, name):
continue
# Get either the file path, module name, or relative path
filename = extract_modulename(section.get("filename", ""),
self.input_dir)
idx = 0
# Use idx to index each test with parameters
for entry in module:
grid = None
# Grid and args cannot both be defined
if "args" in entry and "grid" in entry:
bot.exit(f"{name} has defined both a grid and args.")
# If we find a grid, it has to reference an existing grid
if "grid" in entry and entry["grid"] not in self.grids:
bot.exit(
f"{name} needs grid {entry['grid']} but not found in grids."
)
# If we find a grid, it has to reference an existing grid
if "grid" in entry and entry["grid"] in self.grids:
grid = self.grids[entry["grid"]]
params = deepcopy(entry)
for key in ["grid", "instance"]:
if key in params:
del params[key]
grid.params.update(params)
# A class function is tested over it's instance grid
instance_grid = [{}]
# If entry is defined without a grid, we need to generate it
if not grid:
grid = Grid(name=name,
params=entry,
filename=filename,
refs=self.grids)
# If the grid has an instance, add the correct args to it
if "self" in grid.args and "grid" in grid.args["self"]:
instance_grid = self.grids.get(
grid.args["self"]["grid"], [{}])
# If the grid is cached, we already have parameter sets
argsets = grid
if grid.cache:
argsets = grid.argsets
# iterate over argsets for a grid, get overlapping args
for extra_args in instance_grid:
for argset in argsets:
updated = deepcopy(grid.params)
# Add instance args, if needed
updated["args"] = argset
if extra_args:
updated["args"]["self"] = extra_args
tests["%s.%s" % (name, idx)] = GridTest(
module=parent,
name=name,
params=updated,
verbose=verbose,
cleanup=cleanup,
filename=filename,
show_progress=self.show_progress,
)
print(f"generating test {idx}", end="\r")
idx += 1
return tests
