def test_sb_functional_model(): """ Small tests to show the usage of the sb functional model. """ sb = gen_sb(16, 2, 4, "00", "11", 1, 0, 101010) # TEST #1 # First digit of each element in the array is #side # second digit is #track # This particular pattern is used to debug output incase of wrong result # Last element is random PE output data = [11, 12, 21, 22, 31, 32, 41, 42, 46] # This configuration enables the SB to output the PE output value sb_inst = sb() sb_inst.reset() sb_inst.configure(BitVector(0, 32), BitVector(65535, 32)) res = sb_inst(*data) assert res == [[data[8], data[8]], [data[8], data[8]], [data[8], data[8]], [data[8], data[8]]] # nopep8 # TEST #2 # Random integer inputs to the SB data2 = [randint(0, 10000) for j in range(9)] # This configuration enables the SB to output the 2nd input to the muxes sb_inst2 = sb() sb_inst2.reset() sb_inst2.configure(BitVector(0, 32), BitVector(43690, 32)) res2 = sb_inst2(*data2) assert res2 == [[data2[6], data2[7]], [data2[6], data2[7]], [data2[6], data2[7]], [data2[4], data2[5]]] # nopep8
def value(self): type = libcoreir_c.COREGetValueType(self.ptr) # type enum values defined in include/coreir-c/coreir-args.h if type == 0: return libcoreir_c.COREValueBoolGet(self.ptr) elif type == 1: return libcoreir_c.COREValueIntGet(self.ptr) elif type == 2: if libcoreir_c.COREValueBitVectorIsBinary(self.ptr): from math import ceil width = ct.c_int() libcoreir_c.COREValueBitVectorGetWidth(self.ptr, ct.byref(width)) value_str = ct.create_string_buffer( (str(width.value) + "'h" + "0" * ceil(width.value / 4)).encode()) libcoreir_c.COREValueBitVectorGetString(self.ptr, value_str) prefix, value = value_str.value.split(b"'h") value = int(value, 16) return BitVector(value, num_bits=width.value) else: width = ct.c_int() libcoreir_c.COREValueBitVectorGetWidth(self.ptr, ct.byref(width)) return BitVector(None, num_bits=width.value) elif type == 3: return libcoreir_c.COREValueStringGet(self.ptr).decode() raise NotImplementedError()
def test_print_nested_arrays(capfd, target, simulator): circ = common.TestNestedArraysCircuit actions = [ Poke(circ.I, [BitVector(i, 4) for i in range(3)]), Print(circ.I), Eval(), Expect(circ.O, [BitVector(i, 4) for i in range(3)]), Print(circ.O), Poke(circ.I, [BitVector(4 - i, 4) for i in range(3)]), Eval(), Print(circ.O), ] run(circ, actions, target, simulator, flags=["-Wno-lint"]) out, err = capfd.readouterr() if target == fault.verilator_target.VerilatorTarget: actual = "\n".join(out.splitlines()[-9:]) else: if simulator == "ncsim": actual = "\n".join(out.splitlines()[-9 - 3:-3]) elif simulator == "vcs": actual = "\n".join(out.splitlines()[-9 - 6:-6]) else: raise NotImplementedError(f"Unsupported simulator: {simulator}") assert actual == """\ NestedArraysCircuit.I[0] = 0 NestedArraysCircuit.I[1] = 1 NestedArraysCircuit.I[2] = 2 NestedArraysCircuit.O[0] = 0 NestedArraysCircuit.O[1] = 1 NestedArraysCircuit.O[2] = 2 NestedArraysCircuit.O[0] = 4 NestedArraysCircuit.O[1] = 3 NestedArraysCircuit.O[2] = 2""", out
def advance_clk(self, addr, data): save_stall_reg = self.stall[-1] temp_stall_reg = BitVector(0, self.num_stall_domains) mask = BitVector(addr, self.num_stall_domains) for i in range(self.num_stall_domains): if (mask[i] == 1): temp_stall_reg[i] = 0 self.stall = [temp_stall_reg] * data + [save_stall_reg]
def __init__(self, address_width, data_width): self.address_width = address_width self.data_width = data_width self.data_depth = 1 << address_width self.memory = { BitVector(i, address_width): BitVector(0, data_width) for i in range(self.data_depth) }
def config_write(self, addr, data): rw_delay = self.rw_delay_sel[0] duration = rw_delay.as_uint() self.read = [0] * (duration + 1) self.write = [1] * duration + [0] self.config_addr_out = [BitVector(addr, config_addr_width)] \ * (duration + 1) self.config_data_out = [BitVector(data, config_data_width)] \ * (duration + 1)
def test_tester_nested_arrays(): circ = common.TestNestedArraysCircuit builder = VectorBuilder(circ) expected = [] for i in range(3): val = random.randint(0, (1 << 4) - 1) builder.process(Poke(circ.I[i], BitVector(val, 4))) builder.process(Expect(circ.O[i], BitVector(val, 4))) expected.append(val) assert builder.vectors == [[Array(expected, 3), Array(expected, 3)]]
def test_shift_sim(): Shift = DefineLSL(8, 4) lsl_sim = PythonSimulator(Shift) I = BitVector(0x01, 8) O = BitVector(0x10, 8) lsl_sim.set_value(Shift.I, I) lsl_sim.evaluate() assert lsl_sim.get_value(Shift.O) == O
def test_eq(): assert BitVector(1, 4) == 1 assert BitVector(1, 4) == BitVector(1, 4) assert [BitVector(1, 4)] == [BitVector(1, 4)] assert [[BitVector(1, 4)]] == [[BitVector(1, 4)]] assert BitVector(None, 1) == BitVector(None, 1)
def test_regression(default_value, num_tracks, has_constant): feedthrough_outputs = "1111101111" params = { "width": 16, "num_tracks": num_tracks, "feedthrough_outputs": feedthrough_outputs, "has_constant": has_constant, "default_value": default_value.as_uint() } magma_cb = define_cb(**params) m.compile(f"test_cb/build/{magma_cb.name}", magma_cb, output="coreir-verilog") genesis_cb = cb_wrapper.generator()(**params, input_files=["cb/genesis/cb.vp"]) genesis_verilog = "genesis_verif/cb.v" check_interfaces(magma_cb, genesis_cb) shutil.copy(genesis_verilog, "test_cb/build") config_addr = BitVector(0, 32) cb_functional_model = gen_cb(**params)() class CBTester(ResetTester, ConfigurationTester): pass tester = CBTester(genesis_cb, genesis_cb.clk, cb_functional_model) for config_data in [BitVector(x, 32) for x in range(0, num_tracks)]: tester.zero_inputs() tester.reset() tester.configure(config_addr, config_data) tester.actions += \ generate_actions_from_streams( # Interesting example of Python's dynamic scoping, observe how # the following code is incorrect because of when the string # argument to getattr is evaluated # genesis_cb, cb_functional_model, dict(**{ # f"in_{i}": lambda name, port: random_bv( # len(getattr(genesis_cb, f"in_i{i}"))) # for i in range(num_tracks) if feedthrough_outputs[i] == "1" # }, **{ genesis_cb, cb_functional_model, { f"in_{i}": lambda name, port: random_bv( len(port)) for i in range(num_tracks) if feedthrough_outputs[i] == "1" }) for cb, output in [(genesis_cb, "verilog"), (magma_cb, "coreir-verilog")]: tester.retarget(cb, cb.clk) \ .compile_and_run(directory="test_cb/build", target="verilator", flags=["-Wno-fatal"], magma_output=output)
def __call__(self, op_a=0, op_b=0, c=0, op_d_p=0): bv = UIntVector if not self.signed else SIntVector a = bv(op_a, num_bits=self.width) b = bv(op_b, num_bits=self.width) c = bv(c, num_bits=self.width) d = bv(op_d_p, num_bits=self.width) res = self.op(a, b, c, d) if self._carry: res_p = BitVector([0,0,0,BitVector(a._value + b._value >= (2 ** self.width), 1)]) return res, res_p return res
def test_cb_functional_model(): """ Small test to show the usage of the cb functional model. For now we can leave @num_tracks and @default_value to None. """ cb = gen_cb(16, 4, "1111", False, None) cb_inst = cb() cb_inst.configure(BitVector(0, 32), BitVector(2, 32)) data = [8, 19, 2, 9] res = cb_inst(*data) assert res == data[2]
def write_gc_reg(self, addr, data): if (addr == GCRegAddr.TST_ADDR): self.TST = [BitVector(data, config_data_width)] elif (addr == GCRegAddr.STALL_ADDR): self.stall = [BitVector(data, self.num_stall_domains)] elif (addr == GCRegAddr.CLK_SEL_ADDR): self.clk_sel = [BitVector(data, 1)] elif (addr == GCRegAddr.RW_DELAY_SEL_ADDR): self.rw_delay_sel = [BitVector(data, config_data_width)] elif (addr == GCRegAddr.CLK_SWITCH_DELAY_SEL_ADDR): self.clk_switch_delay_sel = [BitVector(data, 1)] else: raise ValueError("Writing to invalid GC_reg address")
def test_pop_count(): PopCount8 = DefinePopCount(8) m.compile('build/popcount8', PopCount8, output="coreir") assert check_files_equal(__file__, "build/popcount8.json", "gold/popcount8.json") scope = Scope() sim = CoreIRSimulator(PopCount8, None) for I, expected_O in [(1, 1), (2, 1), (3, 2)]: sim.set_value(PopCount8.I, BitVector(I, 8), scope) sim.evaluate() assert BitVector(sim.get_value(PopCount8.O, scope), 8).as_int() == expected_O
def compare(self, a, b, res): eq = a == b eq = eq.as_int() a_msb = msb(a) b_msb = msb(b) c_msb = msb(res) if self.signed: ge = int((~(a_msb ^ b_msb) & ~c_msb) | (~a_msb & b_msb)) & 1 le = int((~(a_msb ^ b_msb) & c_msb) | (a_msb & ~b_msb) | eq) & 1 else: ge = int((~(a_msb ^ b_msb) & ~c_msb) | (a_msb & ~b_msb)) & 1 le = int((~(a_msb ^ b_msb) & c_msb) | (~a_msb & b_msb) | eq) & 1 return BitVector(ge, num_bits=1), \ BitVector(eq, num_bits=1), \ BitVector(le, num_bits=1), \
def test_mux_wrapper(height, width): """ Test that the mux wrapper circuit works as expected. Specifically, we initialize a mux with random height and width, and check that the output is as expected for select in range [0, height). Note that we do not check the behavior with sel >= height, because this is undefined behavior. """ mux = MuxWrapper(height, width) assert mux.height == height assert mux.width == width assert mux.name() == f"MuxWrapper_{height}_{width}" mux_circuit = mux.circuit() tester = fault.Tester(mux_circuit) inputs = [fault.random.random_bv(width) for _ in range(height)] for i, input_ in enumerate(inputs): tester.poke(mux_circuit.I[i], input_) for i in range(height): tester.poke(mux_circuit.S, BitVector(i, mux.sel_bits)) tester.eval() tester.expect(mux_circuit.O, inputs[i]) with tempfile.TemporaryDirectory() as tempdir: tester.compile_and_run(directory=tempdir, magma_output="coreir-verilog", flags=["-Wno-fatal"])
def run(self, actions): simulator = self.backend_cls(self.circuit, self.clock) for action in actions: if isinstance(action, fault.actions.Poke): value = action.value # Python simulator does not support setting Bit with # BitVector(1), so do conversion here if isinstance(action.port, m.BitType) and \ isinstance(value, BitVector): value = value.as_uint() simulator.set_value(action.port, value) elif isinstance(action, fault.actions.Print): got = simulator.get_value(action.port) if isinstance(action.port, m.ArrayType) and \ isinstance(action.port.T, (m._BitType, m._BitKind)): got = BitVector(got).as_uint() elif isinstance(action.port, m.ArrayType): raise NotImplementedError("Printing complex nested arrays") print(f'{action.port.debug_name} = {action.format_str}' % got) elif isinstance(action, fault.actions.Expect): got = simulator.get_value(action.port) expected = action.value if isinstance(expected, fault.actions.Peek): expected = simulator.get_value(expected.port) MagmaSimulatorTarget.check(got, action.port, expected) elif isinstance(action, fault.actions.Eval): simulator.evaluate() elif isinstance(action, fault.actions.Step): if self.clock is not action.clock: raise RuntimeError( f"Using different clocks: {self.clock}, " f"{action.clock}") simulator.advance_cycle(action.steps) else: raise NotImplementedError(action)
def gen_simple_pe(ops: list, data_width: int = 16): CONFIG_DATA_WIDTH = m.bitutils.clog2(len(ops)) CONFIG_ADDR_WIDTH = 1 CONFIG_ADDR = BitVector(0, CONFIG_ADDR_WIDTH) ParentCls = ConfigurableModel(CONFIG_DATA_WIDTH, CONFIG_ADDR_WIDTH) class _SimplePE(ParentCls): def __init__(self): super().__init__() self.ops = ops self.reset() def reset(self): self.O = fault.UnknownValue self.read_data = fault.UnknownValue self.configure(CONFIG_ADDR, BitVector(0, CONFIG_DATA_WIDTH)) def configure(self, addr, data): self.config[addr] = data def __call__(self, I0, I1): select = self.config[CONFIG_ADDR] self.O = self.ops[select.as_uint()](I0, I1) return self.O return _SimplePE
def __call__(self, *args): assert len(args) == num_tracks select = self.config[CONFIG_ADDR] select_as_uint = select.as_uint() if select_as_uint in range(num_tracks): return args[select_as_uint] return BitVector(0, width)
def make_expect(self, i, action): if value_utils.is_any(action.value): return [] if isinstance(action.port, SelectPath): name = f"dut.{action.port.system_verilog_path}" debug_name = action.port[-1].name elif isinstance(action.port, fault.WrappedVerilogInternalPort): name = f"dut.{action.port.path}" debug_name = name else: name = verilog_name(action.port.name) debug_name = action.port.name value = action.value if isinstance(value, actions.Peek): if isinstance(value.port, fault.WrappedVerilogInternalPort): value = f"dut.{value.port.path}" else: value = f"{value.port.name}" elif isinstance(value, PortWrapper): value = f"dut.{value.select_path.system_verilog_path}" elif isinstance(action.port, m.SIntType) and value < 0: # Handle sign extension for verilator since it expects and # unsigned c type port_len = len(action.port) value = BitVector(value, port_len).as_uint() return [ f"if ({name} != {value}) $error(\"Failed on action={i}" f" checking port {debug_name}. Expected %x, got %x\"" f", {value}, {name});" ]
def test_target_clock(capfd, target, simulator): circ = common.TestBasicClkCircuit actions = [ Poke(circ.I, 0), Print(circ.I), Expect(circ.O, 0), Poke(circ.CLK, 0), Print(circ.O), Step(circ.CLK, 1), Poke(circ.I, BitVector(1, 1)), Eval(), Print(circ.O), ] run(circ, actions, target, simulator, flags=["-Wno-lint"]) out, err = capfd.readouterr() lines = out.splitlines() if target == fault.verilator_target.VerilatorTarget: assert lines[-3] == "BasicClkCircuit.I = 0", out assert lines[-2] == "BasicClkCircuit.O = 0", out assert lines[-1] == "BasicClkCircuit.O = 1", out else: if simulator == "ncsim": assert lines[-6] == "BasicClkCircuit.I = 0", out assert lines[-5] == "BasicClkCircuit.O = 0", out assert lines[-4] == "BasicClkCircuit.O = 1", out elif simulator == "vcs": assert lines[-9] == "BasicClkCircuit.I = 0", out assert lines[-8] == "BasicClkCircuit.O = 0", out assert lines[-7] == "BasicClkCircuit.O = 1", out else: raise NotImplementedError(f"Unsupported simulator: {simulator}")
def generate_function_test_vectors(circuit, func, input_ranges=None, mode='complete', flatten=True): check(circuit, func) args = [] for i, (name, port) in enumerate(circuit.IO.items()): if port.isinput(): if isinstance(port, BitKind): args.append([BitVector(0), BitVector(1)]) elif isinstance(port, ArrayKind) and isinstance(port.T, BitKind): num_bits = port.N if isinstance(port, SIntKind): if input_ranges is None: input_range = range(-2**(num_bits - 1), 2**(num_bits - 1)) else: input_range = input_ranges[i] args.append([ SIntVector(x, num_bits=num_bits) for x in input_range ]) else: if input_ranges is None: input_range = range(1 << num_bits) else: input_range = input_ranges[i] args.append( [BitVector(x, num_bits=num_bits) for x in input_range]) else: raise NotImplementedError(type(port)) tests = [] for test in product(*args): result = func(*list(test)) test = [list(test), []] if isinstance(result, tuple): test[-1].extend(result) else: test[-1].append(result) tests.append(test) if flatten: tests = flatten_tests(tests) else: tests = [test[0] + test[1] for test in tests] return tests
def test_print_double_nested_arrays(capfd, target, simulator): circ = common.TestDoubleNestedArraysCircuit actions = [ Poke(circ.I, [[BitVector(i + j * 3, 4) for i in range(3)] for j in range(2)]), Print(circ.I), Eval(), Expect(circ.O, [[BitVector(i + j * 3, 4) for i in range(3)] for j in range(2)]), Print(circ.O), Poke(circ.I, [[BitVector(i + (j + 1) * 3, 4) for i in range(3)] for j in range(2)]), Eval(), Print(circ.O), ] run(circ, actions, target, simulator, flags=["-Wno-lint"]) out, err = capfd.readouterr() print(out) if target == fault.verilator_target.VerilatorTarget: actual = "\n".join(out.splitlines()[-18:]) else: if simulator == "ncsim": actual = "\n".join(out.splitlines()[-18 - 3:-3]) elif simulator == "vcs": actual = "\n".join(out.splitlines()[-18 - 6:-6]) else: raise NotImplementedError(f"Unsupported simulator: {simulator}") assert actual == """\ DoubleNestedArraysCircuit.I[0][0] = 0 DoubleNestedArraysCircuit.I[0][1] = 1 DoubleNestedArraysCircuit.I[0][2] = 2 DoubleNestedArraysCircuit.I[1][0] = 3 DoubleNestedArraysCircuit.I[1][1] = 4 DoubleNestedArraysCircuit.I[1][2] = 5 DoubleNestedArraysCircuit.O[0][0] = 0 DoubleNestedArraysCircuit.O[0][1] = 1 DoubleNestedArraysCircuit.O[0][2] = 2 DoubleNestedArraysCircuit.O[1][0] = 3 DoubleNestedArraysCircuit.O[1][1] = 4 DoubleNestedArraysCircuit.O[1][2] = 5 DoubleNestedArraysCircuit.O[0][0] = 3 DoubleNestedArraysCircuit.O[0][1] = 4 DoubleNestedArraysCircuit.O[0][2] = 5 DoubleNestedArraysCircuit.O[1][0] = 6 DoubleNestedArraysCircuit.O[1][1] = 7 DoubleNestedArraysCircuit.O[1][2] = 8\ """, out
def DefineCoreirConst(width, value): def simulate_coreir_const(self, value_store, state_store): value_store.set_value(self.O, value) return DeclareCoreirCircuit(f"coreir_const{width}{value}", "O", Out(Bits(width)), coreir_name="const", coreir_lib="coreir", coreir_genargs={"width": width}, coreir_configargs={"value": BitVector(value, width)}, simulate=simulate_coreir_const)
def test_mux_with_default_wrapper(num_inputs, width, sel_bits, default): if 2 ** sel_bits <= num_inputs: with pytest.raises(ValueError) as pytest_e: MuxWithDefaultWrapper(num_inputs, width, sel_bits, default) assert False expected_error = ValueError(f"(2 ^ sel_bits) must be > num_inputs " f"(sel_bits={sel_bits}, " f"num_inputs={num_inputs})") assert pytest_e.type == type(expected_error) assert repr(pytest_e.value) == repr(expected_error) return mux = MuxWithDefaultWrapper(num_inputs, width, sel_bits, default) assert mux.num_inputs == num_inputs assert mux.width == width assert mux.sel_bits == sel_bits assert mux.default == default assert mux.name() == f"MuxWithDefaultWrapper_{num_inputs}_{width}"\ f"_{sel_bits}_{default}" mux_circuit = mux.circuit() tester = fault.Tester(mux_circuit) inputs = [fault.random.random_bv(width) for _ in range(num_inputs)] for i, input_ in enumerate(inputs): tester.poke(mux_circuit.I[i], input_) tester.poke(mux_circuit.EN, 1) for i in range(num_inputs): tester.poke(mux_circuit.S, BitVector(i, mux.sel_bits)) tester.eval() tester.expect(mux_circuit.O, inputs[i]) for _ in range(10): sel = choice(range(num_inputs, 2 ** sel_bits)) tester.poke(mux_circuit.S, BitVector(sel, mux.sel_bits)) tester.eval() tester.expect(mux_circuit.O, default) # Test that with EN=0, we get the default value, even with select being in # [0, num_inputs). tester.poke(mux_circuit.EN, 0) for i in range(num_inputs): tester.poke(mux_circuit.S, BitVector(i, mux.sel_bits)) tester.eval() tester.expect(mux_circuit.O, default) with tempfile.TemporaryDirectory() as tempdir: tester.compile_and_run(directory=tempdir, magma_output="coreir-verilog", flags=["-Wno-fatal"])
def test_romb(): main = DefineCircuit("test_romb", "RDATAOUT", Out(Bits(8)), "CLK", In(Clock)) # FIXME: hack romb = ROMB(512, 8, [0b00000001, 0b11111111] + [0] * 510) wire(romb.RADDR, uint(1, 9)) wire(romb.RCLK, main.CLK) wire(romb.RE, 1) wire(romb.RDATA, main.RDATAOUT) EndCircuit() sim = PythonSimulator(main, clock=main.CLK) sim.evaluate() sim.advance(2) assert BitVector(sim.get_value(main.RDATAOUT)) == BitVector(0b11111111, num_bits=8)
def make_bit_vector(N, value): assert isinstance(N, int) if isinstance(value, BitVector) and N == value.num_bits: return value if isinstance(value, int): return BitVector(value, N) if value is AnyValue or value is UnknownValue: return value raise NotImplementedError(N, value)
def make_bit(value): # TODO(rsetaluri): Use bit_vector.Bit when implemented. if isinstance(value, BitVector) and value.num_bits == 1: return value if value == 0 or value == 1: return BitVector(value, 1) if value is AnyValue or value is UnknownValue: return value raise NotImplementedError(value)
def test_map_mulby2(): c = context width = 8 numInputs = 4 module_typ = c.Record({ "in": c.Array(width, c.BitIn()), "out": c.Array(width, c.Bit()) }) mulBy2 = c.global_namespace.new_module("mulBy2", module_typ) mulBy2.print_() mulBy2Def = mulBy2.new_definition() coreir_mul = import_("coreir", "mul") mul = coreir_mul(width=width) mul_inst = mulBy2Def.add_module_instance("mul", mul) mulBy2Def.connect(mulBy2Def.interface.select("in"), mul_inst.select("in0")) const_instantiable = c.get_namespace("coreir").generators["const"] gen_args = c.new_values({"width": width}) config_args = c.new_values({"value": BitVector(8, num_bits=8)}) two = mulBy2Def.add_generator_instance("two", const_instantiable, gen_args, config_args) mulBy2Def.connect(two.select("out"), mul_inst.select("in1")) mulBy2Def.connect(mul_inst.select("out"), mulBy2Def.interface.select("out")) mulBy2.definition = mulBy2Def mapParallelParams = c.new_values({ "numInputs": numInputs, "operator": mulBy2 }) test_module_typ = c.Record({ "in": c.Array(numInputs, c.Array(width, c.BitIn())), "out": c.Array(numInputs, c.Array(width, c.Bit())) }) test_module = c.global_namespace.new_module("test_module", test_module_typ) test_module_def = test_module.new_definition() mapParallel = import_("aetherlinglib", "mapParallel") mapMod = mapParallel(numInputs=numInputs, operator=mulBy2) mapMulBy2 = test_module_def.add_module_instance("mapMulBy2", mapMod) test_module_def.connect(test_module_def.interface.select("in"), mapMulBy2.select("in")) test_module_def.connect(mapMulBy2.select("out"), test_module_def.interface.select("out")) test_module_def.print_() test_module.definition = test_module_def test_module.print_() dir_path = os.path.dirname(os.path.realpath(__file__)) test_module.save_to_file(os.path.join(dir_path, "mapParallel_test.json")) with open(os.path.join(dir_path, "mapParallel_test.json"), "r") as actual: with open(os.path.join(dir_path, "mapParallel_test_gold.json"), "r") as gold: assert actual.read() == gold.read() mod = c.load_from_file(os.path.join(dir_path, "mapParallel_test.json")) mod.print_()
def complete(func, n, width, signed=False, num_args=1): max = 1 << width tests = [] for args in itertools.product(*(range(max) for _ in range(num_args))): test = [BitVector(i, width, signed) for i in args] result = func(*test) test.append(result) tests.append(test) return tests