def __call__(self, x, no_copy=False, no_init=False):
r"""Call transform on the provided message.
Args:
x (object): Message object to transform.
no_copy (bool, optional): If True, the transformation occurs in
place. Otherwise a copy is created and transformed. Defaults
to False.
no_init (bool, optional): If True, the datatype is not initialized
if it is not already set. Defaults to False.
Returns:
object: The transformed message.
"""
if isinstance(x, bytes) and (len(x) == 0) and no_init:
return b''
if isinstance(x, collections.abc.Iterator):
xlist = list(x)
if (not self.original_datatype) and (not no_init) and xlist:
self.set_original_datatype(encode_type(xlist[0]))
out = iter([self.evaluate_transform(xx, no_copy=no_copy)
for xx in xlist])
else:
if (not self.original_datatype) and (not no_init):
self.set_original_datatype(encode_type(x))
out = self.evaluate_transform(x, no_copy=no_copy)
return out
def test_serialize_sinfo(self):
r"""Test serialize/deserialize with serializer info."""
if (self._cls == 'SerializeBase'):
return
hout = copy.deepcopy(self._header_info)
hout.update(self.instance.serializer_info)
temp_seri = import_component(
'serializer', self.instance.serializer_info['seritype'])()
for iobj in self.testing_options['objects']:
hout.update(encode_type(iobj, typedef=self.instance.typedef))
msg = self.instance.serialize(iobj, header_kwargs=self._header_info,
add_serializer_info=True)
iout, ihead = self.instance.deserialize(msg)
hout.update(size=ihead['size'], id=ihead['id'],
incomplete=False)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
# Use info to reconstruct serializer
iout, ihead = temp_seri.deserialize(msg)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
new_seri = import_component('serializer',
ihead.pop('seritype', None))(**ihead)
iout, ihead = new_seri.deserialize(msg)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
def test_serialize_sinfo(self):
r"""Test serialize/deserialize with serializer info."""
if self.testing_options.get('is_user_defined', False):
self.assert_raises(RuntimeError,
self.instance.serialize,
self.testing_options['objects'][0],
add_serializer_info=True)
else:
hout = copy.deepcopy(self._header_info)
hout.update(self.instance.serializer_info)
temp_seri = serialize.get_serializer(
seritype=self.instance.serializer_info['seritype'])
for iobj in self.testing_options['objects']:
hout.update(encode_type(iobj, typedef=self.instance.typedef))
msg = self.instance.serialize(iobj,
header_kwargs=self._header_info,
add_serializer_info=True)
iout, ihead = self.instance.deserialize(msg)
hout.update(size=ihead['size'],
id=ihead['id'],
incomplete=False)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
# Use info to reconstruct serializer
iout, ihead = temp_seri.deserialize(msg)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
new_seri = serialize.get_serializer(**ihead)
iout, ihead = new_seri.deserialize(msg)
self.assert_result_equal(iout, iobj)
self.assert_equal(ihead, hout)
def encode(cls, instance, typedef=None):
r"""Encoder for the 'properties' container property."""
if typedef is None:
typedef = {}
return {
k: encode_type(v, typedef=typedef.get(k, None))
for k, v in instance.items()
}
def setup_model(cls, language, transform, language_ext=None, **kwargs):
r"""Write the model file for the specified combination of
language and type.
Args:
language (str): Language that model should be written in.
transform (str): Transformation that should be performed.
language_ext (str, optional): Extension that should be used
for the model file. If not provided, the extension is
determined from the specified language.
**kwargs: Additional keyword arguments are passed to
the write_function_def class method of the language
driver.
Returns:
tuple(str, dict): Full path to the file that was written
and the environment variables that should be set before
running the integration.
"""
if language_ext is None:
language_ext = get_language_ext(language)
modelfile = os.path.join(_example_dir, cls.example_name, 'src',
'model' + language_ext)
drv = import_component('model', language)
testdata = cls.get_test_data()
testtype = encode_type(testdata)
inputs = [{'name': 'x', 'datatype': copy.deepcopy(testtype)}]
outputs = [{'name': 'y', 'datatype': copy.deepcopy(testtype)}]
# Write the model
function_contents = []
for i, o in zip(inputs, outputs):
function_contents += drv.write_assign_to_output(
o, i, outputs_in_inputs=drv.outputs_in_inputs)
lines = drv.write_function_def('model',
function_contents=function_contents,
inputs=copy.deepcopy(inputs),
outputs=copy.deepcopy(outputs),
outputs_in_inputs=drv.outputs_in_inputs,
opening_msg='IN MODEL',
closing_msg='MODEL EXIT',
print_inputs=True,
print_outputs=True,
**kwargs)
with open(modelfile, 'w') as fd:
print(modelfile)
print('\n'.join(lines))
fd.write('\n'.join(lines))
env = {}
env['TEST_LANGUAGE'] = language
env['TEST_LANGUAGE_EXT'] = language_ext
env['TEST_TRANSFORM'] = transform
if transform == 'table':
env['TEST_MODEL_IO'] = ('outputs:\n' + ' - name: ' +
language + '_model:output\n' +
' format_str: "%s\\t%d\\t%f\\n"')
return modelfile, env
def encode(cls, instance, typedef=None):
r"""Encoder for the 'items' container property."""
if isinstance(typedef, (list, tuple)):
typedef_list = typedef
else:
typedef_list = [copy.deepcopy(typedef) for x in instance]
assert (len(typedef_list) == len(instance))
return [
encode_type(v, typedef=t) for v, t in zip(instance, typedef_list)
]
def test_encode_decode():
r"""Test encode/decode for valid objects."""
for x in _valid_objects.values():
y = datatypes.encode(x)
z = datatypes.decode(y)
assert_equal(z, x)
t = datatypes.encode_type(x)
d = datatypes.encode_data(x)
w = datatypes.decode_data(d, t)
assert_equal(w, x)
assert_raises(ValueError, datatypes.decode_data, b'', None)
def test_encode_decode(nested_approx):
r"""Test encode/decode for valid objects."""
for x in _valid_objects.values():
y = datatypes.encode(x)
z = datatypes.decode(y)
assert (z == nested_approx(x))
t = datatypes.encode_type(x)
d = datatypes.encode_data(x)
w = datatypes.decode_data(d, t)
assert (w == nested_approx(x))
with pytest.raises(ValueError):
datatypes.decode_data(b'', None)
def env(self, example_name, language, transform, example_module):
r"""dict: Environment variables set for the test."""
language_ext = get_language_ext(language)
modelfile = os.path.join(_example_dir, example_name, 'src',
'model' + language_ext)
drv = import_component('model', language)
testdata = example_module.get_test_data(transform)
testtype = encode_type(testdata)
inputs = [{'name': 'x', 'datatype': copy.deepcopy(testtype)}]
outputs = [{'name': 'y', 'datatype': copy.deepcopy(testtype)}]
# Write the model
function_contents = []
for i, o in zip(inputs, outputs):
function_contents += drv.write_assign_to_output(
o, i, outputs_in_inputs=drv.outputs_in_inputs)
lines = drv.write_function_def('model',
function_contents=function_contents,
inputs=copy.deepcopy(inputs),
outputs=copy.deepcopy(outputs),
outputs_in_inputs=drv.outputs_in_inputs,
opening_msg='IN MODEL',
closing_msg='MODEL EXIT',
print_inputs=True,
print_outputs=True)
with open(modelfile, 'w') as fd:
print(modelfile)
print('\n'.join(lines))
fd.write('\n'.join(lines))
env = {}
env['TEST_LANGUAGE'] = language
env['TEST_LANGUAGE_EXT'] = language_ext
env['TEST_TRANSFORM'] = transform
if transform == 'table':
env['TEST_MODEL_IO'] = ('outputs:\n' + ' - name: ' +
language + '_model:output\n' +
' format_str: "%s\\t%d\\t%f\\n"')
try:
yield env
finally:
if os.path.isfile(modelfile):
os.remove(modelfile)
def transform_datatype(self, datatype):
r"""Determine the datatype that will result from applying the transform
to the supplied datatype.
Args:
datatype (dict): Datatype to transform.
Returns:
dict: Transformed datatype.
"""
try:
out = encode_type(self(generate_data(datatype)))
if (((out['type'] == 'array') and (datatype['type'] == 'array')
and isinstance(out['items'], list)
and isinstance(datatype['items'], list)
and (len(out['items']) == len(datatype['items'])))):
for x, y in zip(out['items'], datatype['items']):
if 'title' in y:
x.setdefault('title', y['title'])
return out
except NotImplementedError: # pragma: debug
return datatype
def encode(cls, instance, typedef=None):
r"""Encoder for the 'temptype' property."""
return encode_type(instance, typedef)
def setup_model(cls, language, typename, language_ext=None,
using_pointers=False, using_generics=False,
split_array=False, dont_add_lengths=False,
length_prefix=False, assign_kws=None, **kwargs):
r"""Write the model file for the specified combination of
language and type.
Args:
language (str): Language that model should be written in.
typename (str): Type that should be expected by the model.
language_ext (str, optional): Extension that should be used
for the model file. If not provided, the extension is
determined from the specified language.
using_pointers (bool, optional): If True and the tested
language supports pointers, pointers will be used rather
than explicit arrays. Defaults to False.
using_generics (bool, optional): If True and the tested
language has a dedicated generic class, the generic
type will be used rather than explict types. Defaults
to False.
split_array (bool, optional): If True and the tested datatype
is an array, the variables will be split and specified
explicitly in the yaml. Defaults to False.
dont_add_lengths (bool, optional): If True, lengths will not
be added to the definition or assignments. Defaults to
False.
length_prefix (bool, optional): If True, the length variables
will be given prefixes instead of suffixes. Defaults to
False.
assign_kws (dict, optional): Keyword arguments for the calls
to write_assign_to_output. Defaults to {}.
**kwargs: Additional keyword arguments are passed to
the write_function_def class method of the language
driver.
Returns:
str: Full path to the file that was written.
"""
if assign_kws is None:
assign_kws = {}
if language in ['c', 'c++', 'cpp']:
# dont_add_lengths is only valid for C/C++
kwargs['dont_add_lengths'] = dont_add_lengths
kwargs['use_length_prefix'] = length_prefix
assign_kws.setdefault('dont_add_lengths', dont_add_lengths)
assign_kws.setdefault('use_length_prefix', length_prefix)
yaml_fields = {'vars': False, 'dtype': False}
if language_ext is None:
language_ext = get_language_ext(language)
modelfile = os.path.join(_example_dir, cls.example_name,
'src', 'model' + language_ext)
drv = import_component('model', language)
if using_generics and drv.is_typed:
testtype = {'type': 'any'}
else:
testdata = cls.get_test_data(typename)
testtype = encode_type(testdata)
using_generics = False
if split_array and (typename == 'array'):
inputs = [{'name': 'x%d' % i, 'datatype': x} for i, x in
enumerate(copy.deepcopy(testtype['items']))]
outputs = [{'name': 'y%d' % i, 'datatype': x} for i, x in
enumerate(copy.deepcopy(testtype['items']))]
else:
inputs = [{'name': 'x',
'datatype': copy.deepcopy(testtype)}]
outputs = [{'name': 'y',
'datatype': copy.deepcopy(testtype)}]
# Write the model
function_contents = []
for i, o in zip(inputs, outputs):
if using_pointers and drv.is_typed:
for k in ['shape', 'length']:
i['datatype'].pop(k, None)
o['datatype'].pop(k, None)
function_contents += drv.write_assign_to_output(
o, i, outputs_in_inputs=drv.outputs_in_inputs,
**assign_kws)
lines = drv.write_function_def(
'model', function_contents=function_contents,
inputs=copy.deepcopy(inputs),
outputs=copy.deepcopy(outputs),
outputs_in_inputs=drv.outputs_in_inputs,
opening_msg='IN MODEL', closing_msg='MODEL EXIT',
print_inputs=True, print_outputs=True, **kwargs)
with open(modelfile, 'w') as fd:
print(modelfile)
print('\n'.join(lines))
fd.write('\n'.join(lines))
os.environ['TEST_LANGUAGE'] = language
os.environ['TEST_LANGUAGE_EXT'] = language_ext
os.environ['TEST_TYPENAME'] = typename
if (language in ['c', 'fortran']) and (not using_generics):
yaml_fields['vars'] = True
if typename in ['array', 'object']:
yaml_fields['dtype'] = True
if any(list(yaml_fields.values())):
lines = []
for io, io_vars in zip(['input', 'output'],
[inputs, outputs]):
lines += [io + 's:',
' name: %s' % io]
if yaml_fields['vars']:
lines.append(
' vars: %s' % cls.get_varstr(
io_vars, language,
using_pointers=using_pointers,
length_prefix=length_prefix,
dont_add_lengths=dont_add_lengths))
if yaml_fields['dtype']:
if len(io_vars) == 1:
dtype = io_vars[0]['datatype']
else:
dtype = {'type': 'array',
'items': [x['datatype'] for
x in io_vars]}
lines.append(' datatype:')
for x in yaml.dump(dtype).splitlines():
if "units: ''" in x:
continue
lines.append(' ' + x)
os.environ['TEST_MODEL_IO'] = '\n '.join(lines) + '\n'
else:
os.environ['TEST_MODEL_IO'] = ''
return modelfile
def env(self, example_name, language, typename, using_pointers,
using_generics, split_array, dont_add_lengths, length_prefix,
example_module):
r"""dict: Environment variables set for the test."""
kwargs = {}
assign_kws = {}
if language in ['c', 'c++', 'cpp']:
# dont_add_lengths is only valid for C/C++
kwargs['dont_add_lengths'] = dont_add_lengths
kwargs['use_length_prefix'] = length_prefix
assign_kws.setdefault('dont_add_lengths', dont_add_lengths)
assign_kws.setdefault('use_length_prefix', length_prefix)
yaml_fields = {'vars': False, 'dtype': False}
language_ext = get_language_ext(language)
modelfile = os.path.join(os.path.dirname(__file__), example_name,
'src', 'model' + language_ext)
drv = import_component('model', language)
if using_generics and drv.is_typed:
testtype = {'type': 'any'}
else:
testdata = example_module.get_test_data(typename)
testtype = encode_type(testdata)
using_generics = False
if split_array and (typename == 'array'):
inputs = [{
'name': 'x%d' % i,
'datatype': x
} for i, x in enumerate(copy.deepcopy(testtype['items']))]
outputs = [{
'name': 'y%d' % i,
'datatype': x
} for i, x in enumerate(copy.deepcopy(testtype['items']))]
else:
inputs = [{'name': 'x', 'datatype': copy.deepcopy(testtype)}]
outputs = [{'name': 'y', 'datatype': copy.deepcopy(testtype)}]
# Write the model
function_contents = []
for i, o in zip(inputs, outputs):
if using_pointers and drv.is_typed:
for k in ['shape', 'length']:
i['datatype'].pop(k, None)
o['datatype'].pop(k, None)
function_contents += drv.write_assign_to_output(
o, i, outputs_in_inputs=drv.outputs_in_inputs, **assign_kws)
lines = drv.write_function_def('model',
function_contents=function_contents,
inputs=copy.deepcopy(inputs),
outputs=copy.deepcopy(outputs),
outputs_in_inputs=drv.outputs_in_inputs,
opening_msg='IN MODEL',
closing_msg='MODEL EXIT',
print_inputs=True,
print_outputs=True,
**kwargs)
with open(modelfile, 'w') as fd:
print(modelfile)
print('\n'.join(lines))
fd.write('\n'.join(lines))
env = {}
env['TEST_LANGUAGE'] = language
env['TEST_LANGUAGE_EXT'] = language_ext
env['TEST_TYPENAME'] = typename
if (language in ['c', 'fortran']) and (not using_generics):
yaml_fields['vars'] = True
if typename in ['array', 'object']:
yaml_fields['dtype'] = True
if any(list(yaml_fields.values())):
lines = []
for io, io_vars in zip(['input', 'output'], [inputs, outputs]):
lines += [io + 's:', ' name: %s' % io]
if yaml_fields['vars']:
lines.append(
' vars: %s' %
self.get_varstr(io_vars,
language,
using_pointers=using_pointers,
length_prefix=length_prefix,
dont_add_lengths=dont_add_lengths))
if yaml_fields['dtype']:
if len(io_vars) == 1:
dtype = io_vars[0]['datatype']
else:
dtype = {
'type': 'array',
'items': [x['datatype'] for x in io_vars]
}
lines.append(' datatype:')
for x in yaml.dump(dtype).splitlines():
if "units: ''" in x:
continue
lines.append(' ' + x)
env['TEST_MODEL_IO'] = '\n '.join(lines) + '\n'
else:
env['TEST_MODEL_IO'] = ''
try:
yield env
finally:
if os.path.isfile(modelfile):
os.remove(modelfile)
