def sparsetest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.exact): if state['sparse'] == enum_to_string(JacobianFormat.sparse): if platform_is_gpu(state['platform']): assert state['vecsize'] == 64 else: assert state['vecsize'] == 4 assert state['order'] == 'F' else: if platform_is_gpu(state['platform']): assert state['order'] == 'C' assert state['vecsize'] == 128 else: assert state['vecsize'] == 2
def get_overrides(test, eval_type, jac_type, sparse_type): """ Convenience method to extract overrides from the given test Parameters ---------- test: dict The test with specified overrides eval_type: str The stringified :class:`build_type` jac_type: str The stringified :class:`JacobianType` sparse_type: str The stringified :class:`JacobianFormat` """ if eval_type == enum_to_string(build_type.species_rates): if eval_type in test: return test[eval_type].copy() return {} else: # first check for the base type overrides = {} if jac_type in test: if sparse_type in test[jac_type]: overrides.update(test[jac_type][sparse_type]) # next check for "both" if 'both' in test[jac_type]: # note: these are guarenteed not to collide, as they've been # previously checked overrides.update(test[jac_type]['both']) return overrides.copy()
def update_jactype(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert state['num_cores'] == 1 assert state['width'] is None assert state['depth'] is None assert state['order'] == 'C' assert state['conp'] is True else: assert state['width'] in [4, None]
def __test_limit(enumlist, limit): stypes = [enum_to_string(enum) for enum in listify(enumlist)] root = models['TestMech']['limits'] for i, stype in enumerate(stypes): assert stype in root if i == len(stypes) - 1: assert root[stype] == limit else: root = root[stype]
def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): if state['platform'] == 'openmp': assert not bool(state['width']) else: assert state['width'] == 128
def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert set(state['models']) == set(['H2']) else: assert set(state['models']) == set(['H2', 'CH4'])
def test_get_test_matrix(): # test that the example test matrix is specified correctly def update(want, state, key, seen): if state[key] not in seen[key]: if six.callable(want[key]): want[key](state, want, seen) else: if is_iterable(state[key]): for k in state[key]: if k not in seen[key]: want[key].remove(k) seen[key].add(k) else: want[key].remove(state[key]) seen[key].add(state[key]) return want, seen def run(want, loop, final_checks=None): from copy import deepcopy seen = defaultdict(lambda: set()) test = deepcopy(want) for state in loop: for key in test: update(test, state, key, seen) # assert we didn't miss anything (that isn't callable -- those handle # themselves) assert not any(len(v) for v in test.values() if not six.callable(v)) if final_checks: assert final_checks(seen) import logging logger = logging.getLogger(__name__) logger.debug('loading test matrix schema') test_matrix = __prefixify('test_matrix.yaml', examples_dir) # get the species validation test logger.debug('loading test matrix from file') _, loop, max_vec_width = get_test_matrix('.', KernelType.species_rates, test_matrix, True, langs=current_test_langs, raise_on_missing=True) assert max_vec_width == 8 from collections import defaultdict def width_check(state, want, seen): if state['lang'] == 'c': assert state['width'] is None assert state['depth'] is None else: seen['width'].add(state['width']) def check_final_widths(seen): return not (set(seen['width']) - set([None, 2, 4, 8])) # check we have reasonable values base = { 'platform': ['intel', 'openmp'], 'width': width_check, 'conp': [True, False], 'order': ['C', 'F'], 'num_cores': num_cores_default()[0] } logger.debug('check 1') run(base, loop, final_checks=check_final_widths) # repeat for jacobian logger.debug('loading test matrix from file [1]') _, loop, _ = get_test_matrix('.', KernelType.jacobian, test_matrix, True, langs=current_test_langs, raise_on_missing=True) jacbase = base.copy() jacbase.update({ 'jac_format': [ enum_to_string(JacobianFormat.sparse), enum_to_string(JacobianFormat.full) ], 'jac_type': [enum_to_string(JacobianType.exact)], 'use_atomic_doubles': [True, False] }) # true for OpenMP by default logger.debug('check 2') run(jacbase, loop, final_checks=check_final_widths) # next, do species performance logger.debug('loading test matrix from file [2]') _, loop, _ = get_test_matrix('.', KernelType.species_rates, test_matrix, False, langs=current_test_langs, raise_on_missing=True) want = base.copy() want.update({'order': ['F']}) logger.debug('check 3') run(want, loop, final_checks=check_final_widths) # and finally, the Jacobian performance logger.debug('loading test matrix from file [4]') _, loop, _ = get_test_matrix('.', KernelType.jacobian, test_matrix, False, langs=current_test_langs, raise_on_missing=True) want = jacbase.copy() # no more openmp want.update({'use_atomic_doubles': [False]}) def update_jactype(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert state['num_cores'] == 1 assert state['width'] is None assert state['depth'] is None assert state['order'] == 'C' assert state['conp'] is True else: assert state['width'] in [4, None] want.update({'platform': ['intel'], 'jac_type': update_jactype}) def check_final_widths(seen): return len(seen['width'] - set([4, None])) == 0 logger.debug('check 5') run(want, loop, final_checks=check_final_widths) # test gpu vs cpu specs logger.debug('writing temp file') with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti platform-list: - name: nvidia lang: opencl width: [128] - name: intel lang: opencl width: [4] test-list: - test-type: performance eval-type: jacobian exact: sparse: gpuwidth: [64] order: ['F'] full: width: [2] gpuorder: ['C'] """)) file.flush() logger.debug('loading test matrix from file [5]') _, loop, _ = get_test_matrix('.', KernelType.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) from pyjac.utils import platform_is_gpu def sparsetest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.exact): if state['jac_format'] == enum_to_string(JacobianFormat.sparse): if platform_is_gpu(state['platform']): assert state['width'] in [64, None] else: assert state['width'] in [4, None] assert state['order'] == 'F' else: if platform_is_gpu(state['platform']): assert state['order'] == 'C' assert state['width'] in [128, None] else: assert state['width'] in [2, None] want = {'jac_format': sparsetest} logger.debug('check 6') run(want, loop) # test model override logger.debug('writing temp file 2') with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl width: [128] test-list: - test-type: performance eval-type: jacobian finite_difference: both: models: ['H2'] """)) file.flush() logger.debug('loading test matrix from file [6]') _, loop, _ = get_test_matrix('.', KernelType.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert set(state['models']) == set(['H2']) else: assert set(state['models']) == set(['H2', 'CH4']) want = {'models': modeltest} logger.debug('check 7') run(want, loop) # finally test bad model spec logger.debug('writing temp file 3') with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl width: [128] test-list: - test-type: performance eval-type: jacobian finite_difference: both: models: ['BAD'] """)) file.flush() logger.debug('loading test matrix from file [7]') with assert_raises(InvalidOverrideException): get_test_matrix('.', KernelType.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) # test gpu vectype specification logger.debug('writing temp file 4') with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl width: [128] - name: openmp lang: c test-list: - test-type: performance eval-type: jacobian finite_difference: both: width: [] depth: [] gpuwidth: [128] """)) file.flush() logger.debug('loading test matrix from file [8]') _, loop, _ = get_test_matrix('.', KernelType.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): if state['platform'] == 'openmp': assert not bool(state['width']) else: assert state['width'] == 128 want = {'models': modeltest} logger.debug('check 8') run(want, loop) # test that source terms evaluations don't inherit exact jacobian overrides logger.debug('writing temp file 5') with NamedTemporaryFile(mode='w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 mech: gri30.cti path: platform-list: - lang: c name: openmp test-list: - test-type: performance # limit to intel platforms: [openmp] eval-type: both exact: both: num_cores: [1] order: [F] gpuorder: [C] conp: [conp] width: [2, 4] gpuwidth: [128] models: [] """)) file.flush() logger.debug('loading test matrix from file [9]') _, loop, _ = get_test_matrix('.', KernelType.species_rates, file.name, False, langs=current_test_langs, raise_on_missing=True) want = { 'platform': ['openmp'], 'width': [None], 'conp': [True, False], 'order': ['C', 'F'], 'models': ['CH4'], 'num_cores': num_cores_default()[0] } logger.debug('check 9') run(want, loop)
def __run_test(self, method, test_python_wrapper=True, ktype=KernelType.species_rates, **oploop_keywords): kwargs = {} if not test_python_wrapper: kwargs['shared'] = [True, False] oploop_keywords.update(kwargs) ignored_state_vals = ['conp'] + list(kwargs.keys()) wrapper = OptionLoopWrapper.from_get_oploop( self, ignored_state_vals=ignored_state_vals, do_conp=False, **oploop_keywords) for opts in wrapper: with temporary_build_dirs() as (build_dir, obj_dir, lib_dir): # write files # write files conp = wrapper.state['conp'] kgen = method(self.store.reacs, self.store.specs, opts, conp=conp) # generate kgen.generate(build_dir, species_names=[x.name for x in self.store.specs], rxn_strings=[str(x) for x in self.store.reacs]) # write header write_aux(build_dir, opts, self.store.specs, self.store.reacs) if test_python_wrapper: package = 'pyjac_{}'.format(utils.package_lang[opts.lang]) # test wrapper generation pywrap(opts.lang, build_dir, obj_dir=obj_dir, out_dir=lib_dir, ktype=ktype) imp = test_utils.get_import_source() with open(os.path.join(lib_dir, 'test_import.py'), 'w') as file: file.write( imp.substitute( path=lib_dir, package=package, kernel=utils.enum_to_string(ktype).title(), nsp=len(self.store.specs), nrxn=len(self.store.reacs))) utils.run_with_our_python( [os.path.join(lib_dir, 'test_import.py')]) else: # compile generate_library(opts.lang, build_dir, obj_dir=obj_dir, out_dir=lib_dir, shared=wrapper.state['shared'], ktype=ktype)
def get_test_matrix(work_dir, test_type, test_matrix, for_validation, raise_on_missing=True, langs=get_test_langs()): """Runs a set of mechanisms and an ordered dictionary for performance and functional testing Parameters ---------- work_dir : str Working directory with mechanisms and for data test_type: :class:`build_type.jacobian` Controls some testing options (e.g., whether to do a sparse matrix or not) test_matrix: str The test matrix file to load for_validation: bool If determines which test type to load from the test matrix, validation or performance raise_on_missing: bool Raise an exception of the specified :param:`test_matrix` file is not found langs: list of str The allowed languages, modifiable by the :envvar:`TEST_LANGS` or test_langs in :file:`test_setup.py` Returns ------- mechanisms : dict A dictionary indicating which mechanism are available for testing, The structure is as follows: mech_name : {'mech' : file path to the Cantera mechanism 'ns' : number of species in the mechanism 'limits' : {'full': XXX, 'sparse': XXX}}: a dictionary of limits on the number of conditions that can be evaluated for this mechanism (full & sparse jacobian respectively) due to memory constraints params : OrderedDict The parameters to put in an oploop max_vec_width : int The maximum vector width to test """ work_dir = abspath(work_dir) # validate the test matrix matrix_name = test_matrix test_matrix = build_and_validate('test_matrix_schema.yaml', test_matrix) # check that we have the working directory if not exists(work_dir): raise Exception('Work directory {} for '.format(work_dir) + 'testing not found, exiting...') # load the models models = load_models(work_dir, test_matrix) assert isinstance(test_type, build_type) # load tests tests = load_tests(test_matrix, matrix_name) # filter those that match the test type valid_str = 'validation' if for_validation else 'performance' tests = [test for test in tests if test['test-type'] == valid_str] tests = [ test for test in tests if test['eval-type'] == enum_to_string(test_type) or test['eval-type'] == 'both' ] # and dictify tests = [OrderedDict(test) for test in tests] if not tests: raise Exception('No tests found in matrix {} for {} test of {}, ' 'exiting...'.format(matrix_name, valid_str, enum_to_string(test_type))) # get defaults we haven't migrated to schema yet rate_spec = ['fixed', 'hybrid'] if test_type != build_type.jacobian \ else ['fixed'] sparse = ([ enum_to_string(JacobianFormat.sparse), enum_to_string(JacobianFormat.full) ] if test_type == build_type.jacobian else [enum_to_string(JacobianFormat.full)]) jac_types = [ enum_to_string(JacobianType.exact), enum_to_string(JacobianType.finite_difference) ] if (test_type == build_type.jacobian and not for_validation) else [enum_to_string(JacobianType.exact)] split_kernels = [False] # and default # of cores, this may be overriden default_num_cores, can_override_cores = num_cores_default() # load platforms platforms = load_platforms(test_matrix, langs=langs, raise_on_empty=raise_on_missing) platforms = [OrderedDict(platform) for platform in platforms] out_params = [] logger = logging.getLogger(__name__) for test in tests: # filter platforms plats = [p.copy() for p in platforms] if 'platforms' in test: plats = [ plat for plat in plats if plat['platform'] in test['platforms'] ] if len(plats) < len(platforms): logger.info( 'Platforms ({}) filtered out for test type: {}'.format( ', '.join([ p['platform'] for p in platforms if p not in plats ]), ' - '.join([test['test-type'], test['eval-type']]))) if not len(plats): logger.warn('No platforms found for test {}, skipping...'.format( ' - '.join([test['test-type'], test['eval-type']]))) continue for plookup in plats: clean = plookup.copy() # get default number of cores cores = default_num_cores[:] # get default vector widths widths = plookup['width'] is_wide = widths is not None depths = plookup['depth'] is_deep = depths is not None if is_deep and not is_wide: widths = depths[:] # sanity check if is_wide or is_deep: assert widths is not None # special gpu handling for cores is_gpu = False # test platform type if platform_is_gpu(plookup['platform']): # set cores to 1 is_gpu = True cores = [1] def apply_vectypes(lookup, widths, is_wide=is_wide, is_deep=is_deep): if is_wide or is_deep: # set vec widths use_par = None in widths or (is_wide and is_deep) lookup['vecsize'] = [x for x in widths[:] if x is not None] base = [True] if not use_par else [True, False] if is_wide: lookup['wide'] = base[:] base.pop() if is_deep: lookup['deep'] = base[:] else: lookup['vecsize'] = [None] lookup['wide'] = [False] lookup['deep'] = [False] del lookup['width'] del lookup['depth'] apply_vectypes(plookup, widths) # default is both conp / conv conp = [True, False] order = ['C', 'F'] # loop over possible overrides oploop = OptionLoop( OrderedDict([('ttype', [enum_to_string(test_type)]), ('jtype', jac_types), ('stype', sparse)])) for i, state in enumerate(oploop): ttype = state['ttype'] jtype = state['jtype'] stype = state['stype'] def override_log(key, old, new): logging.info( 'Replacing {} for test type: {}. Old value:' ' ({}), New value: ({})'.format( key, stringify_args( [ttype, test['eval-type'], jtype, stype], joiner='.'), stringify_args(listify(old)), stringify_args(listify(new)))) # copy defaults icores = cores[:] iorder = order[:] iconp = conp[:] ivecsizes = widths[:] if widths is not None else [None] imodels = tuple(models.keys()) # load overides overrides = get_overrides(test, ttype, jtype, stype) # check that we can apply if 'num_cores' in overrides and not can_override_cores: raise InvalidTestEnivironmentException( ttype, 'num_cores', matrix_name, 'num_threads') elif 'num_cores' in overrides and is_gpu: logger = logging.getLogger(__name__) logger.info( 'Discarding unused "num_cores" override for GPU ' 'platform {}'.format(plookup['platform'])) del overrides['num_cores'] # 'num_cores', 'order', 'conp', 'vecsize', 'vectype' # now apply overrides outplat = plookup.copy() for current in overrides: ivectypes_override = None for override in overrides: if override == 'num_cores': override_log('num_cores', icores, overrides[override]) icores = overrides[override] elif override == 'order' and not is_gpu: override_log('order', iorder, overrides[override]) iorder = overrides[override] elif override == 'gpuorder' and is_gpu: override_log('order', iorder, overrides[override]) iorder = overrides[override] elif override == 'conp': iconp_save = iconp[:] iconp = [] if 'conp' in overrides[override]: iconp.append(True) if 'conv' in overrides[override]: iconp.append(False) override_log('conp', iconp_save, iconp) elif override == 'vecsize' and not is_gpu: override_log('vecsize', ivecsizes, overrides[override]) outplat['vecsize'] = listify(overrides[override]) elif override == 'gpuvecsize' and is_gpu: override_log('gpuvecsize', ivecsizes, overrides[override]) outplat['vecsize'] = listify(overrides[override]) elif override == 'vectype' and not is_gpu: # we have to do this at the end ivectypes_override = overrides[override] elif override == 'gpuvectype' and is_gpu: ivectypes_override = overrides[override] elif override == 'models': # check that all models are valid for model in overrides[override]: if model not in imodels: raise InvalidOverrideException( override, model, imodels) # and replace override_log('models', stringify_args(imodels), stringify_args(overrides[override])) imodels = tuple(overrides[override]) if ivectypes_override is not None: c = clean.copy() apply_vectypes(c, outplat['vecsize'], is_wide='wide' in ivectypes_override, is_deep='deep' in ivectypes_override) # and copy into working outplat['wide'] = c['wide'] if 'wide' in c else [False] outplat['deep'] = c['deep'] if 'deep' in c else [False] outplat['vecsize'] = c['vecsize'] old = [''] if is_wide: old += ['wide'] if is_deep: old += ['deep'] elif not is_wide: old += ['par'] override_log('vecsize', old, ivectypes_override) # and finally, convert back to an option loop format out_params.append( [('num_cores', icores), ('order', iorder), ('rate_spec', rate_spec), ('split_kernels', split_kernels), ('conp', iconp), ('sparse', [stype]), ('jac_type', [jtype]), ('models', [imodels])] + [(key, value) for key, value in six.iteritems(outplat)]) max_vec_width = 1 vector_params = [ dict(p)['vecsize'] for p in out_params if 'vecsize' in dict(p) and dict(p)['vecsize'] != [None] ] if vector_params: max_vec_width = max(max_vec_width, max([max(x) for x in vector_params])) from . import reduce_oploop loop = reduce_oploop(out_params) return models, loop, max_vec_width
def load_tests(matrix, filename): """ Load the tests from the pre-parsed test matrix file and check for duplicates Parameters ---------- matrix: dict The parsed test matrix, i.e., output of :func:`build_and_validate` Returns ------- tests: list The list of valid tests """ tests = matrix[test_matrix_key] dupes = set() def __getenumtype(ttype): from argparse import ArgumentTypeError try: EnumType(JacobianType)(ttype) return JacobianType except ArgumentTypeError: try: EnumType(build_type)(ttype) return build_type except: EnumType(JacobianFormat)(ttype) return JacobianFormat from collections import defaultdict for test in tests: # first check that the descriptors = [test['test-type'] + ' - ' + test['eval-type']] if test['eval-type'] == 'both': descriptors = [ test['test-type'] + ' - ' + enum_to_string(build_type.jacobian), test['test-type'] + ' - ' + enum_to_string(build_type.species_rates) ] if set(descriptors) & dupes: raise DuplicateTestException(test['test-type'], test['eval-type'], filename) dupes.update(descriptors) # now go through overrides def roverride_check(root, keydict, dupes, path=''): def __basecheck(k, o, d): if isinstance(k[o], list): # the next level down is the base, hence add a list if o not in d: d[o] = [] if path: path += '.' # find the keys in the root for override in [k for k in keydict if k in root]: # if the override is a 'both' type, apply to all subvalues if override == 'both': for val in keydict[override]: __basecheck(keydict[override], val, dupes) roverride_check(root[override], keydict[override][val], dupes[val], path + val) # test if we've reached the base of the tree if isinstance(keydict, list): # we've reached the base if override not in keydict: # this should be handled by validation, but just double check raise UnknownOverrideException(override, path) if override in dupes: # check for collision raise OverrideCollisionException(override, path) # finally, just add the override dupes.append(override) else: __basecheck(keydict, override, dupes) # we're still on a leaf roverride_check(root[override], keydict[override], dupes[override], path + override) # overrides only need to be checked within a test nesteddict = lambda: defaultdict(nesteddict) # noqa roverride_check(test, allowed_override_keys, nesteddict()) return tests
for p in platform_list: # create option loop and add oploop += [inner_loop + p] elif lang == 'c': inner_loop += [('platform', 'OpenMP')] oploop += [inner_loop] return oploop # todo -- feed these directly into override schema allowed_overrides = [ 'num_cores', 'gpuorder', 'order', 'conp', 'vecsize', 'vectype', 'gpuvecsize', 'gpuvectype', 'models' ] jacobian_sub_override_keys = { enum_to_string(JacobianFormat.sparse): allowed_overrides, enum_to_string(JacobianFormat.full): allowed_overrides, } jacobian_sub_override_keys['both'] = jacobian_sub_override_keys.copy() allowed_override_keys = { enum_to_string(JacobianType.exact): jacobian_sub_override_keys, enum_to_string(JacobianType.finite_difference): jacobian_sub_override_keys, enum_to_string(build_type.species_rates): allowed_overrides } # and add handlers here @nottest def load_tests(matrix, filename): """ Load the tests from the pre-parsed test matrix file and check for duplicates
def test_enum_to_string(enum, string): assert enum_to_string(enum) == string
def test_get_test_matrix(): # test that the example test matrix is specified correctly def update(want, state, key, seen): if state[key] not in seen[key]: if six.callable(want[key]): want[key](state, want, seen) else: if is_iterable(state[key]): for k in state[key]: if k not in seen[key]: want[key].remove(k) seen[key].add(k) else: want[key].remove(state[key]) seen[key].add(state[key]) return want, seen def run(want, loop, final_checks=None): from copy import deepcopy seen = defaultdict(lambda: set()) test = deepcopy(want) for state in loop: for key in test: update(test, state, key, seen) # assert we didn't miss anything (that isn't callable -- those handle # themselves) assert not any(len(v) for v in test.values() if not six.callable(v)) if final_checks: assert final_checks(seen) test_matrix = __prefixify('test_matrix.yaml', examples_dir) # get the species validation test _, loop, max_vec_width = get_test_matrix('.', build_type.species_rates, test_matrix, True, langs=current_test_langs, raise_on_missing=True) assert max_vec_width == 8 from collections import defaultdict def vecsize_check(state, want, seen): if state['lang'] == 'c': assert state['vecsize'] is None assert state['wide'] is False assert state['deep'] is False else: seen['vecsize'].add(state['vecsize']) def check_final_vecsizes(seen): return sorted(seen['vecsize']) == [2, 4, 8] # check we have reasonable values base = { 'platform': ['intel', 'openmp'], 'wide': [True, False], 'vecsize': vecsize_check, 'conp': [True, False], 'order': ['C', 'F'], 'num_cores': num_cores_default()[0] } run(base, loop, final_checks=check_final_vecsizes) # repeat for jacobian _, loop, _ = get_test_matrix('.', build_type.jacobian, test_matrix, True, langs=current_test_langs, raise_on_missing=True) jacbase = base.copy() jacbase.update({ 'sparse': [ enum_to_string(JacobianFormat.sparse), enum_to_string(JacobianFormat.full) ], 'jac_type': [enum_to_string(JacobianType.exact)], 'use_atomics': [True, False] }) # true for OpenMP by default run(jacbase, loop, final_checks=check_final_vecsizes) # next, do species performance _, loop, _ = get_test_matrix('.', build_type.species_rates, test_matrix, False, langs=current_test_langs, raise_on_missing=True) want = base.copy() want.update({'order': ['F']}) run(want, loop, final_checks=check_final_vecsizes) # and finally, the Jacobian performance _, loop, _ = get_test_matrix('.', build_type.jacobian, test_matrix, False, langs=current_test_langs, raise_on_missing=True) want = jacbase.copy() # no more openmp want.update({'use_atomics': [False]}) def update_jactype(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert state['num_cores'] == 1 assert state['vecsize'] is None assert state['wide'] is False assert state['depth'] is False assert state['order'] == 'C' assert state['conp'] is True else: assert state['vecsize'] == 4 want.update({'platform': ['intel'], 'jac_type': update_jactype}) def check_final_vecsizes(seen): return len(seen['vecsize'] - set([4, None])) == 0 run(want, loop, final_checks=check_final_vecsizes) # test gpu vs cpu specs with NamedTemporaryFile('w', suffix='.yaml') as file: file.write(""" model-list: - name: CH4 path: mech: gri30.cti platform-list: - name: nvidia lang: opencl vectype: [wide] vecsize: [128] - name: intel lang: opencl vectype: [wide] vecsize: [4] test-list: - test-type: performance eval-type: jacobian exact: sparse: gpuvecsize: [64] order: ['F'] full: vecsize: [2] gpuorder: ['C'] """) file.flush() _, loop, _ = get_test_matrix('.', build_type.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) from pyjac.tests import platform_is_gpu def sparsetest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.exact): if state['sparse'] == enum_to_string(JacobianFormat.sparse): if platform_is_gpu(state['platform']): assert state['vecsize'] == 64 else: assert state['vecsize'] == 4 assert state['order'] == 'F' else: if platform_is_gpu(state['platform']): assert state['order'] == 'C' assert state['vecsize'] == 128 else: assert state['vecsize'] == 2 want = {'sparse': sparsetest} run(want, loop) # test model override with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl vectype: [wide] vecsize: [128] test-list: - test-type: performance eval-type: jacobian finite_difference: both: models: ['H2'] """)) file.flush() _, loop, _ = get_test_matrix('.', build_type.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): assert set(state['models']) == set(['H2']) else: assert set(state['models']) == set(['H2', 'CH4']) want = {'models': modeltest} run(want, loop) # finally test bad model spec with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl vectype: [wide] vecsize: [128] test-list: - test-type: performance eval-type: jacobian finite_difference: both: models: ['BAD'] """)) file.flush() with assert_raises(InvalidOverrideException): get_test_matrix('.', build_type.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) # test gpu vectype specification with NamedTemporaryFile('w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 path: mech: gri30.cti - name: H2 path: mech: h2o2.cti platform-list: - name: nvidia lang: opencl vectype: [wide, par] vecsize: [128] - name: openmp lang: c vectype: [par] test-list: - test-type: performance eval-type: jacobian finite_difference: both: vectype: [par] gpuvectype: [wide] """)) file.flush() _, loop, _ = get_test_matrix('.', build_type.jacobian, file.name, False, langs=current_test_langs, raise_on_missing=True) def modeltest(state, want, seen): if state['jac_type'] == enum_to_string(JacobianType.finite_difference): if state['platform'] == 'openmp': assert not bool(state['vecsize']) else: assert state['vecsize'] == 128 want = {'models': modeltest} run(want, loop) # test that source terms evaluations don't inherit exact jacobian overrides with NamedTemporaryFile(mode='w', suffix='.yaml') as file: file.write( remove_common_indentation(""" model-list: - name: CH4 mech: gri30.cti path: platform-list: - lang: c name: openmp vectype: ['par'] test-list: - test-type: performance # limit to intel platforms: [openmp] eval-type: both exact: both: num_cores: [1] order: [F] gpuorder: [C] conp: [conp] vecsize: [2, 4] gpuvecsize: [128] gpuvectype: [wide] vectype: [wide] models: [] """)) file.flush() _, loop, _ = get_test_matrix('.', build_type.species_rates, file.name, False, langs=current_test_langs, raise_on_missing=True) want = { 'platform': ['openmp'], 'wide': [False], 'vecsize': [None], 'conp': [True, False], 'order': ['C', 'F'], 'models': ['CH4'], 'num_cores': num_cores_default()[0] } run(want, loop)
def generate_wrapper(lang, pyxfile, build_dir, ktype=KernelType.jacobian, additional_inputs=[], additional_outputs=[], nice_name=None): """ Generate the Cython wrapper file Parameters ---------- lang : str The language of the wrapper being generated pyxfile : str Filename of the pyx file template build_dir : str The path to place the generated cython wrapper in ktype : :class:`KernelType` [KernelType.jacobian] The type of wrapper to generate additional_inputs : list of str If supplied, treat these arguments as additional input variables additional_outputs : list of str If supplied, treat these arguments as additional output variables nice_name: str [None] If supplied, use this instead of :param:`ktype` to derive the kernel name Returns ------- wrapper: str The path to the generated python wrapper """ # create wrappergen if nice_name is None: nice_name = utils.enum_to_string(ktype) if ktype == KernelType.jacobian: inputs, outputs = jac_args(True) # replace 'P_arr' w/ 'param' for clarity replacements = {'P_arr': 'param'} elif ktype != KernelType.dummy: inputs, outputs = rate_args(True, ktype) replacements = { 'cp': 'specific_heat', 'cv': 'specific_heat', 'h': 'specific_energy', 'u': 'specific_energy' } else: assert additional_outputs assert additional_inputs replacements = {} inputs = additional_inputs[:] outputs = additional_outputs[:] def extend(names, args=[]): for name in names: if name in replacements: name = replacements[name] if name not in args: args.append(name) return args args = extend(outputs, extend(inputs)) wrapper = WrapperGen(name=nice_name, kernel_args=args, lang=lang) # dump wrapper with utils.temporary_directory() as tdir: wrappergen = os.path.join(tdir, 'wrappergen.pickle') with open(wrappergen, 'wb') as file: pickle.dump(wrapper, file) infile = pyxfile outfile = 'pyjac_{}.pyx'.format(utils.package_lang[lang]) outfile = os.path.join(build_dir, outfile) # and cogify try: Cog().callableMain([ 'cogapp', '-e', '-d', '-Dwrappergen={}'.format(wrappergen), '-o', outfile, infile ]) except Exception: logger = logging.getLogger(__name__) logger.error( 'Error generating python wrapper file: {}'.format(outfile)) raise return outfile