def main(functions_or_sigs, results_file, cores, debug):
"""
Attempt to build all the signatures in functions_or_sigs, or all the signatures
associated with all the functions in functions_or_sigs, or if functions_or_sigs
is empty every signature the stanc3 compiler exposes.
Results are written to a results json file. Individual signatures are classified
as either compatible, incompatible, or irrelevant.
Compatible signatures can be compiled with varmat types in every argument that
could possibly be a varmat (the matrix-like ones).
Incompatible signatures cannot all be built, and for irrelevant signatures it does
not make sense to try to build them (there are no matrix arguments, or the function
does not support reverse mode autodiff, etc).
Compilation is done in parallel using the number of specified cores.
:param functions_or_sigs: List of function names and/or signatures to benchmark
:param results_file: File to use as a results cache
:param cores: Number of cores to use for compiling
:param debug: If true, don't delete temporary files
"""
all_signatures = get_signatures()
functions, signatures = handle_function_list(functions_or_sigs)
requested_functions = set(functions)
compatible_signatures = set()
incompatible_signatures = set()
irrelevant_signatures = set()
# Read the arguments and figure out the exact list of signatures to test
signatures_to_check = set()
for signature in all_signatures:
sp = SignatureParser(signature)
if len(requested_functions
) > 0 and sp.function_name not in requested_functions:
continue
signatures_to_check.add(signature)
work_queue = Queue.Queue()
# For each signature, generate cpp code to test
for signature in signatures_to_check:
sp = SignatureParser(signature)
if sp.is_high_order():
work_queue.put((n, signature, None))
continue
cpp_code = ""
any_overload_uses_varmat = False
for m, overloads in enumerate(
itertools.product(("Prim", "Rev", "RevVarmat"),
repeat=sp.number_arguments())):
cg = CodeGenerator()
arg_list_base = cg.build_arguments(sp, overloads, size=1)
arg_list = []
for overload, arg in zip(overloads, arg_list_base):
if arg.is_reverse_mode() and arg.is_varmat_compatible(
) and overload.endswith("Varmat"):
any_overload_uses_varmat = True
arg = cg.to_var_value(arg)
arg_list.append(arg)
cg.function_call_assign("stan::math::" + sp.function_name,
*arg_list)
cpp_code += TEST_TEMPLATE.format(
test_name=sp.function_name + repr(m),
code=cg.cpp(),
)
if any_overload_uses_varmat:
work_queue.put((work_queue.qsize(), signature, cpp_code))
else:
print("{0} ... Irrelevant".format(signature.strip()))
irrelevant_signatures.add(signature)
output_lock = threading.Lock()
if not os.path.exists(WORKING_FOLDER):
os.mkdir(WORKING_FOLDER)
work_queue_original_length = work_queue.qsize()
# Test if each cpp file builds and update the output file
# This part is done in parallel
def worker():
while True:
try:
n, signature, cpp_code = work_queue.get(False)
except Queue.Empty:
return # If queue is empty, worker quits
# Use signature as filename prefix to make it easier to find
prefix = re.sub('[^0-9a-zA-Z]+', '_', signature.strip())
# Test the signature
successful = build_signature(prefix, cpp_code, debug)
# Acquire a lock to do I/O
with output_lock:
if successful:
result_string = "Success"
compatible_signatures.add(signature)
else:
result_string = "Fail"
incompatible_signatures.add(signature)
print("Results of test {0} / {1}, {2} ... ".format(
n, work_queue_original_length, signature.strip()) +
result_string)
work_queue.task_done()
for i in range(cores):
threading.Thread(target=worker).start()
work_queue.join()
with open(results_file, "w") as f:
json.dump(
{
"compatible_signatures": list(compatible_signatures),
"incompatible_signatures": list(incompatible_signatures),
"irrelevant_signatures": list(irrelevant_signatures)
},
f,
indent=4,
sort_keys=True)
def main(functions=(), j=1):
"""
Generates expression tests.
For every signature prim, rev and fwd instantiations are tested (with all scalars
of type double/var/fvar<double>). Tests check the following:
- signatures can be compiled with expressions arguments
- results when given expressions are same as when given plain matrices
(including derivatives)
- functions evaluate expressions at most once
:param functions: functions to generate tests for. This can contain names of functions
already supported by stanc3, full function signatures or file names of files containing
any of the previous two. Default: all signatures supported by stanc3
:param j: number of files to split tests in
"""
tests = []
functions, signatures = handle_function_list(functions)
signatures = set(signatures)
if not functions and not signatures:
default_checks = True
signatures |= set(get_signatures())
else:
for signature in get_signatures():
sp = SignatureParser(signature)
if sp.function_name in functions:
signatures.add(signature)
default_checks = False
for n, signature in enumerate(signatures):
for overload in ("Prim", "Rev", "Fwd"):
sp = SignatureParser(signature)
# skip ignored signatures if running the default checks
if default_checks and sp.is_ignored():
continue
# skip signatures without inputs that can be eigen types
if not sp.has_eigen_compatible_arg():
continue
# skip functions in forward mode with no forward mode overload, same for reverse mode
if overload == "Fwd" and not sp.is_fwd_compatible():
continue
if overload == "Rev" and not sp.is_rev_compatible():
continue
is_reverse_mode = overload == "Rev" and not sp.returns_int()
cg = CodeGenerator()
# Generate two sets of arguments, one will be expressions one will not
arg_list_expression_base = cg.build_arguments(
sp, sp.number_arguments() * [overload], size=1)
arg_list_expression = [
cg.convert_to_expression(arg, size=1)
if arg.is_eigen_compatible() else arg
for arg in arg_list_expression_base
]
arg_list_no_expression = cg.build_arguments(sp,
sp.number_arguments() *
[overload],
size=1)
# Check results with expressions and without are the same
cpp_function_name = "stan::math::" + sp.function_name
result = cg.function_call_assign(cpp_function_name,
*arg_list_expression)
result_no_expression = cg.function_call_assign(
cpp_function_name, *arg_list_no_expression)
cg.expect_eq(result, result_no_expression)
# Check that expressions evaluated only once
for arg in arg_list_expression:
if arg.is_expression():
cg.expect_leq_one(arg.counter)
# If reverse mode, check the adjoints
if is_reverse_mode:
summed_result = cg.recursive_sum(result)
summed_result_no_expression = cg.recursive_sum(
result_no_expression)
sum_of_sums = cg.add(summed_result,
summed_result_no_expression)
cg.grad(sum_of_sums)
for arg, arg_no_expression in zip(arg_list_no_expression,
arg_list_expression):
cg.expect_adj_eq(arg, arg_no_expression)
cg.recover_memory()
tests.append(
test_code_template.format(
overload=overload,
comment=signature.strip(),
test_name=sp.function_name + repr(n),
code=cg.cpp(),
))
save_tests_in_files(j, tests)
class CodeGeneratorTest(unittest.TestCase):
def setUp(self):
self.add = SignatureParser("add(real, vector) => vector")
self.int_var = IntVariable("myint", 5)
self.real_var1 = RealVariable("Rev", "myreal1", 0.5)
self.real_var2 = RealVariable("Rev", "myreal2", 0.5)
self.matrix_var = MatrixVariable("Rev", "mymatrix", "matrix", 2, 0.5)
self.cg = CodeGenerator()
def test_prim_prim(self):
self.cg.build_arguments(self.add, ["Prim", "Prim"], 1)
self.assertEqual(
self.cg.cpp(), """double real0 = 0.4;
auto matrix1 = stan::test::make_arg<Eigen::Matrix<double, Eigen::Dynamic, 1>>(0.4, 1);"""
)
def test_prim_rev(self):
self.cg.build_arguments(self.add, ["Prim", "Rev"], 1)
self.assertEqual(
self.cg.cpp(), """double real0 = 0.4;
auto matrix1 = stan::test::make_arg<Eigen::Matrix<stan::math::var, Eigen::Dynamic, 1>>(0.4, 1);"""
)
def test_rev_rev(self):
self.cg.build_arguments(self.add, ["Rev", "Rev"], 1)
self.assertEqual(
self.cg.cpp(), """stan::math::var real0 = 0.4;
auto matrix1 = stan::test::make_arg<Eigen::Matrix<stan::math::var, Eigen::Dynamic, 1>>(0.4, 1);"""
)
def test_size(self):
self.cg.build_arguments(self.add, ["Rev", "Rev"], 2)
self.assertEqual(
self.cg.cpp(), """stan::math::var real0 = 0.4;
auto matrix1 = stan::test::make_arg<Eigen::Matrix<stan::math::var, Eigen::Dynamic, 1>>(0.4, 2);"""
)
def test_add(self):
self.cg.add(self.real_var1, self.real_var2)
self.assertEqual(
self.cg.cpp(),
"auto sum_of_sums0 = stan::math::eval(stan::math::add(myreal1,myreal2));"
)
def test_convert_to_expression(self):
self.cg.convert_to_expression(self.matrix_var)
self.assertEqual(
self.cg.cpp(), """int mymatrix_expr0_counter = 0;
stan::test::counterOp<stan::math::var> mymatrix_expr0_counter_op(&mymatrix_expr0_counter);
auto mymatrix_expr0 = mymatrix.block(0,0,2,2).unaryExpr(mymatrix_expr0_counter_op);"""
)
def test_expect_adj_eq(self):
self.cg.expect_adj_eq(self.real_var1, self.real_var2)
self.assertEqual(self.cg.cpp(),
"stan::test::expect_adj_eq(myreal1,myreal2);")
def test_expect_eq(self):
self.cg.expect_eq(self.real_var1, self.real_var2)
self.assertEqual(self.cg.cpp(), "EXPECT_STAN_EQ(myreal1,myreal2);")
def test_expect_leq_one(self):
self.cg.expect_leq_one(self.int_var)
self.assertEqual(self.cg.cpp(), """int int0 = 1;
EXPECT_LE(myint,int0);""")
def test_function_call_assign(self):
self.cg.function_call_assign("stan::math::add", self.real_var1,
self.real_var2)
self.assertEqual(
self.cg.cpp(),
"auto result0 = stan::math::eval(stan::math::add(myreal1,myreal2));"
)
def test_grad(self):
self.cg.grad(self.real_var1)
self.assertEqual(self.cg.cpp(), "stan::test::grad(myreal1);")
def test_recover_memory(self):
self.cg.recover_memory()
self.assertEqual(self.cg.cpp(), "stan::math::recover_memory();")
def test_recursive_sum(self):
self.cg.recursive_sum(self.real_var1)
self.assertEqual(
self.cg.cpp(),
"auto summed_result0 = stan::math::eval(stan::test::recursive_sum(myreal1));"
)
def test_to_var_value(self):
self.cg.to_var_value(self.matrix_var)
self.assertEqual(
self.cg.cpp(),
"auto mymatrix_varmat0 = stan::math::eval(stan::math::to_var_value(mymatrix));"
)
