def test_compilation_generator(self):
# currently separate compiler code only exists for OpenCL
oploop = OptionLoopWrapper.from_get_oploop(self,
langs=['opencl'],
do_conp=False,
do_vector=False,
do_sparse=False)
for opts in oploop:
callgen = CallgenResult(
source_names=['adistinctivetestname', 'andyetanothertestname'])
# create a species rates kernel generator for this state
kgen = get_jacobian_kernel(self.store.reacs,
self.store.specs,
opts,
conp=oploop.state['conp'])
with temporary_directory() as tdir:
comp = kgen._generate_compiling_program(tdir, callgen)
file = os.path.join(
tdir, kgen.name + '_compiler' + file_ext[opts.lang])
with open(file, 'r') as file:
comp = file.read()
# test filenames
assert '"adistinctivetestname", "andyetanothertestname"' in comp
# test build options
assert kgen._get_cl_level() in comp
# outname
assert 'char* out_name = "{}";'.format(kgen.name + '.bin')
# and platform
assert 'char* platform = "{}";'.format(opts.platform.vendor)
def test_unique_pointer_specification(self):
with utils.temporary_directory() as build_dir:
# test good fixed size
create_jacobian('c',
gas=self.store.gas,
unique_pointers=True,
data_order='F',
build_path=build_dir,
kernel_type=KernelType.species_rates)
files = ['species_rates', 'chem_utils']
files = [f + ext for f in files for ext in [utils.file_ext['c']]]
for file in files:
# read resulting file
with open(os.path.join(build_dir, file), 'r') as file:
file = file.read()
# and make sure we don't have 'work_size
assert not re.search(r'\b{}\b'.format(work_size.name), file)
def test_subkernel_pointers(self):
# this test is designed to ensure that we get the corrected / expected
# signature / pointer unwrapping in subkernels
oploop = OptionLoopWrapper.from_get_oploop(self,
do_conp=False,
do_vector=True,
do_sparse=False)
for opts in oploop:
# create generators
kgen = self._kernel_gen(opts)
with temporary_directory() as tdir:
kgen._make_kernels()
codegen_results = kgen._generate_wrapping_kernel(
tdir, return_codegen_results=True)
# first, check the pointer unpacks for each kgen
dep, top = codegen_results
# check that the dependent kernel only has the 'pointer unpack #1'
# array
assert len(dep.pointer_unpacks) == 1 and 'punpack1' in \
dep.pointer_offsets
# check that the top level kernel has pointer unpacks #1 & #2, as
# well as 'arg', which is neither an input or output
unpacks = 3 if not opts.depth else 4
assert len(top.pointer_unpacks) == unpacks and all([
x in top.pointer_offsets
for x in ['punpack1', 'punpack2', 'arg']
])
# and check that the offset is the same as in dep
assert top.pointer_offsets['punpack1'] == dep.pointer_offsets[
'punpack1']
# and finally, check the signature of the dependent kernel gen
# via the kernel
(owner, dep) = kgen._generate_wrapping_kernel(
tdir, return_memgen_records=True)
dep_kd = set([x.name for x in dep.kernel_data])
own_kd = set([x.name for x in owner.kernel_data])
assert 'arg1' not in [x.name for x in dep.kernel_data]
# and test that any working buffers in the owner are in the subrecord
for wbuf in [int_work_name, local_work_name, rhs_work_name]:
assert (wbuf in dep_kd) == (wbuf in own_kd)
def test_can_load():
"""
Tests whether the external cog code-gen app can load our serialized objects
"""
wrapper = __test_cases()
for opts in wrapper:
# create a dummy callgen
callgen = CallgenResult(order=opts.order, lang=opts.lang,
dev_mem_type=wrapper.state['dev_mem_type'],
type_map=type_map(opts.lang))
with temporary_directory() as tdir:
with open(os.path.join(tdir, 'test.cpp'), mode='w') as file:
file.write("""
/*[[[cog
import cog
import os
import pickle
# next, unserialize the callgen
with open(callgen, 'rb') as file:
call = pickle.load(file)
# and create a memory manager
from pyjac.kernel_utils.memory_tools import get_memory
mem = get_memory(call)
cog.outl('success!')
]]]
[[[end]]]*/""")
# and serialize mem
with open(os.path.join(tdir, 'callgen.pickle'), 'wb') as file:
pickle.dump(callgen, file)
# and call cog
from cogapp import Cog
cmd = [
'cog', '-e', '-d', '-Dcallgen={}'.format(
os.path.join(tdir, 'callgen.pickle')),
'-o', os.path.join(tdir, 'test'), os.path.join(tdir, 'test.cpp')]
Cog().callableMain(cmd)
with open(os.path.join(tdir, 'test'), 'r') as file:
assert file.read().strip() == 'success!'
def generate_setup(lang,
setupfile,
pyxfile,
home_dir,
build_dir,
out_dir,
libname,
extra_include_dirs=[],
libraries=[],
libdirs=[],
ktype=KernelType.jacobian):
"""Helper method to fill in the template .in files
Parameters
----------
lang : str
The language of the wrapper being generated
setupfile : str
Filename of the setup file template
pyxfile : str
Filename of the pyx file template
home_dir : str
Home directory path
build_dir : str
Build directory path
out_dir : str
Output directory path
libname : str
Library name
extra_include_dirs : Optional[list of str]
Optional; if supplied, extra include directions for the python wrapper
libraries : Optional[list of str]
Optional; if supplied extra libraries to use
libdirs : Optional[list of str]
Optional; if supplied, library directories
Returns
-------
setup: str
The path to the generated setup.py file
"""
setup = SetupGen(name='pyjac',
libname=libname,
include_dirs=extra_include_dirs,
package_lang=utils.package_lang[lang],
wrapper=pyxfile,
lang=lang,
build_dir=build_dir,
libraries=libraries,
libdirs=libdirs)
# serialize
# dump wrapper
with utils.temporary_directory() as tdir:
setupgen = os.path.join(tdir, 'setupgen.pickle')
with open(setupgen, 'wb') as file:
pickle.dump(setup, file)
infile = setupfile
outfile = os.path.basename(infile[:infile.rindex('.in')])
outfile = os.path.join(out_dir, outfile)
# and cogify
try:
Cog().callableMain([
'cogapp', '-e', '-d', '-Dsetupgen={}'.format(setupgen), '-o',
outfile, infile
])
except Exception:
logger = logging.getLogger(__name__)
logger.error(
'Error generating python setup file: {}'.format(outfile))
raise
return outfile
def generate_wrapper(lang,
pyxfile,
build_dir,
ktype=KernelType.jacobian,
additional_inputs=[],
additional_outputs=[],
nice_name=None):
"""
Generate the Cython wrapper file
Parameters
----------
lang : str
The language of the wrapper being generated
pyxfile : str
Filename of the pyx file template
build_dir : str
The path to place the generated cython wrapper in
ktype : :class:`KernelType` [KernelType.jacobian]
The type of wrapper to generate
additional_inputs : list of str
If supplied, treat these arguments as additional input variables
additional_outputs : list of str
If supplied, treat these arguments as additional output variables
nice_name: str [None]
If supplied, use this instead of :param:`ktype` to derive the kernel name
Returns
-------
wrapper: str
The path to the generated python wrapper
"""
# create wrappergen
if nice_name is None:
nice_name = utils.enum_to_string(ktype)
if ktype == KernelType.jacobian:
inputs, outputs = jac_args(True)
# replace 'P_arr' w/ 'param' for clarity
replacements = {'P_arr': 'param'}
elif ktype != KernelType.dummy:
inputs, outputs = rate_args(True, ktype)
replacements = {
'cp': 'specific_heat',
'cv': 'specific_heat',
'h': 'specific_energy',
'u': 'specific_energy'
}
else:
assert additional_outputs
assert additional_inputs
replacements = {}
inputs = additional_inputs[:]
outputs = additional_outputs[:]
def extend(names, args=[]):
for name in names:
if name in replacements:
name = replacements[name]
if name not in args:
args.append(name)
return args
args = extend(outputs, extend(inputs))
wrapper = WrapperGen(name=nice_name, kernel_args=args, lang=lang)
# dump wrapper
with utils.temporary_directory() as tdir:
wrappergen = os.path.join(tdir, 'wrappergen.pickle')
with open(wrappergen, 'wb') as file:
pickle.dump(wrapper, file)
infile = pyxfile
outfile = 'pyjac_{}.pyx'.format(utils.package_lang[lang])
outfile = os.path.join(build_dir, outfile)
# and cogify
try:
Cog().callableMain([
'cogapp', '-e', '-d', '-Dwrappergen={}'.format(wrappergen),
'-o', outfile, infile
])
except Exception:
logger = logging.getLogger(__name__)
logger.error(
'Error generating python wrapper file: {}'.format(outfile))
raise
return outfile
def test_read_initial_condition_generator(self):
oploop = OptionLoopWrapper.from_get_oploop(self,
do_conp=False,
do_vector=True,
do_sparse=False)
for opts in oploop:
# two kernels (one for each generator)
spec_insns = ("""
{spec} = {param} {{id=0}}
""")
param_insns = ("""
{param} = 1 {{id=1}}
""")
# create mapstore
domain = arc.creator('domain',
arc.kint_type, (10, ),
'C',
initializer=np.arange(10,
dtype=arc.kint_type))
mapstore = arc.MapStore(opts, domain, None)
# create global args
param = arc.creator(arc.pressure_array, np.float64,
(arc.problem_size.name, 10), opts.order)
spec = arc.creator('longanddistinct', np.float64,
(arc.problem_size.name, 10), opts.order)
namestore = type('', (object, ), {
'param': param,
'spec': spec,
'jac': ''
})
# create array / array strings
param_lp, param_str = mapstore.apply_maps(param, 'j', 'i')
spec_lp, spec_str = mapstore.apply_maps(spec, 'j', 'i')
# create kernel infos
spec_info = knl_info(
'spec_eval',
spec_insns.format(param=param_str, spec=spec_str),
mapstore,
kernel_data=[spec_lp, param_lp, arc.work_size],
silenced_warnings=['write_race(0)'])
param_info = knl_info('param_eval',
param_insns.format(param=param_str),
mapstore,
kernel_data=[param_lp, arc.work_size],
silenced_warnings=['write_race(1)'])
# create generators
param_gen = make_kernel_generator(opts,
KernelType.chem_utils,
[param_info],
namestore,
output_arrays=[param.name])
spec_gen = make_kernel_generator(
opts,
KernelType.species_rates, [spec_info],
namestore,
depends_on=[param_gen],
input_arrays=[spec.name, param.name],
output_arrays=[spec.name])
# get the record
with temporary_directory() as tdir:
spec_gen._make_kernels()
_, record, _ = spec_gen._generate_wrapping_kernel(tdir)
# and call the read IC gen
spec_gen._generate_common(tdir, record)
# read in header
with open(
os.path.join(
tdir,
'read_initial_conditions' + header_ext[opts.lang]),
'r') as file:
file = file.read()
assert 'double* longanddistinct' in file
# read in source
with open(
os.path.join(
tdir,
'read_initial_conditions' + file_ext[opts.lang]),
'r') as file:
file = file.read()
assert 'double* longanddistinct' in file
def test_call_header_generator(self):
oploop = OptionLoopWrapper.from_get_oploop(self,
do_conp=False,
do_vector=True,
do_sparse=False)
for opts in oploop:
with temporary_directory() as tdir:
jac_gen = self.__get_call_kernel_generator(opts)
jac_gen._make_kernels()
callgen, record, result = jac_gen._generate_wrapping_kernel(
tdir)
callgen = jac_gen._generate_driver_kernel(
tdir, record, result, callgen)
_, callgen = jac_gen._generate_calling_program(
tdir, 'dummy.bin', callgen, record, for_validation=True)
file = jac_gen._generate_calling_header(tdir, callgen)
with open(file, 'r') as file:
file_src = file.read()
assert 'JacobianKernel();' in file_src
assert (
'JacobianKernel(size_t problem_size, size_t work_size, '
'bool do_not_compile=false);') in file_src
assert 'void operator()(double* h_jac, double* h_spec);'
headers = ['mechanism', 'error_check', 'timer']
if can_vectorize_lang[opts.lang]:
headers += ['vectorization']
assert all(
'#include "{}"'.format(header +
header_ext[opts.lang]) in file_src
for header in headers)
assert re.search(r'static const char\* _order;', file_src)
assert re.search(r'static const unsigned int _nsp;', file_src)
assert re.search(r'static const unsigned int _nrxn;', file_src)
assert re.search(r'static const char\* _order;', file_src)
if opts.lang == 'opencl':
# check build options
assert re.search(r'static const char\* build_options;',
file_src)
assert re.search(r'static const char\* platform_check;',
file_src)
assert r'static const unsigned int device_type;' in file_src
assert r'static const unsigned int _vector_width;' in file_src
# check arguments
for x in jac_gen.in_arrays + jac_gen.out_arrays:
assert re.search(r'cl_mem d_{};'.format(x), file_src)
# and work arrays
for x in callgen.work_arrays:
if not isinstance(x, lp.ValueArg):
try:
name = x.name
except AttributeError:
name = x
assert re.search(r'cl_mem d_{};'.format(name),
file_src)
else:
assert 'CL_LEVEL' not in file_src
for x in jac_gen.in_arrays + jac_gen.out_arrays:
assert re.search(r'double\* d_{};'.format(x), file_src)
# and work arrays
for x in callgen.work_arrays:
if not isinstance(x, lp.ValueArg):
try:
name = x.name
except AttributeError:
name = x
assert re.search(r'double\* d_{};'.format(name),
file_src)
def test_call_generator(self):
oploop = OptionLoopWrapper.from_get_oploop(self,
do_conp=False,
do_vector=True,
do_sparse=False)
for opts in oploop:
with temporary_directory() as tdir:
jac_gen = self.__get_call_kernel_generator(opts)
jac_gen._make_kernels()
callgen, record, result = jac_gen._generate_wrapping_kernel(
tdir)
callgen = jac_gen._generate_driver_kernel(
tdir, record, result, callgen)
out, _ = jac_gen._generate_calling_program(tdir,
'dummy.bin',
callgen,
record,
for_validation=True)
# check that 1) it runs
with open(out, 'r') as file:
file_src = file.read()
# check for local defn's
for arr in jac_gen.in_arrays + jac_gen.out_arrays:
assert 'double* h_{}_local;'.format(arr) in file_src
# check for operator defn
assert ('void JacobianKernel::operator()(double* h_spec, '
'double* h_jac)') in file_src
assert re.search(r'const char\* Kernel::_order =', file_src)
assert re.search(r'const unsigned int Kernel::_nsp =',
file_src)
assert re.search(r'const unsigned int Kernel::_nrxn =',
file_src)
if opts.lang == 'c':
assert ('void Kernel::threadset(unsigned int num_threads)'
in file_src)
else:
# check build options
assert re.search(
r'const char\* Kernel::build_options = "[^\n]+-I{}'.
format(re.escape(tdir)), file_src)
assert re.search(
r'const char\* Kernel::platform_check = "{}";'.format(
re.escape(opts.platform.vendor)), file_src)
assert r'const unsigned int Kernel::device_type = ' in \
file_src
assert r'const unsigned int Kernel::_vector_width = ' in file_src
# and the validation output
assert all(x.strip() in file_src
for x in """// write output to file if supplied
char* output_files[1] = {"jac.bin"};
size_t output_sizes[1] = {10 * problem_size};
double* outputs[1] = {h_jac_local};
for(int i = 0; i < 1; ++i)
{
write_data(output_files[i], outputs[i], output_sizes[i]);
}
""".split('\n'))
# check that kernel arg order matches expected
assert [x.name for x in callgen.kernel_args['jacobian']
] == ['spec', 'jac']
# and check that the call matches the kernel order
with open(next(x for x in callgen.source_names if 'jac' in x),
'r') as file:
file_src = file.read()
if opts.lang == 'c':
assert (
'void jacobian(double const *__restrict__ t, '
'double const *__restrict__ spec, '
'double *__restrict__ jac, double *__restrict__ rwk)'
) in file_src
elif opts.lang == 'opencl':
dtype = 'double' if not opts.is_simd else 'double{}'.format(
opts.vector_width)
assert ('jacobian(__global double const *__restrict__ t, '
'__global {dtype} const *__restrict__ spec, '
'__global {dtype} *__restrict__ jac, '
'__global double *__restrict__ rwk)'.format(
dtype=dtype)) in file_src