def test_get_units(self):
r"""Test get_units."""
for v in self._vars_nounits:
self.assert_equal(units.get_units(v), '')
for v in self._vars_units:
self.assert_equal(units.get_units(v),
str(units.as_unit('cm').units))
def assertUnitsEqual(self, first, second, msg=None):
r"""Assertion for equality in case of objects with units."""
if units.has_units(first) and units.has_units(second):
first = units.convert_to(first, units.get_units(second))
self.assertEqual(units.get_data(first),
units.get_data(second),
msg=msg)
def concatenate(cls, objects, as_array=False, **kwargs):
r"""Concatenate objects to get object that would be recieved if
the concatenated serialization were deserialized.
Args:
objects (list): Objects to be concatenated.
as_array (bool, optional): If True, the objects in the list
are complete columns in a table and as_format is set to True.
Defaults to False.
**kwargs: Additional keyword arguments are ignored.
Returns:
list: Set of objects that results from concatenating those provided.
"""
if len(objects) == 0:
return []
if as_array:
units_list = [units.get_units(ix) for ix in objects[0]]
out = [[
units.add_units(np.hstack([x[i] for x in objects]), u)
for i, u in enumerate(units_list)
]]
elif isinstance(objects[0], bytes):
out = [b''.join(objects)]
else:
return super(DefaultSerialize, cls).concatenate(objects, **kwargs)
return out
def _file_send(self, msg):
assert (isinstance(msg, dict))
for k, v in msg.items():
if isinstance(v, np.ndarray):
dims = []
v = self.transform_type_send(v)
for i, d in enumerate(v.shape):
if i == 0:
idim = k
else:
idim = '%s%d' % (k, i)
dims.append(idim)
self._fd_netcdf.createDimension(idim, d)
var = self._fd_netcdf.createVariable(k, v.dtype, dims)
var[:] = v
if units.has_units(v):
var.units = units.get_units(v)
else: # pragma: debug
raise TypeError("Type '%s' no supported." % type(msg))
self._last_size = self.file_size
self.fd.seek(self._last_size)
def assert_equal(x, y):
r"""Assert that two messages are equivalent.
Args:
x (object): Python object to compare against y.
y (object): Python object to compare against x.
Raises:
AssertionError: If the two messages are not equivalent.
"""
if isinstance(y, (list, tuple)):
assert(isinstance(x, (list, tuple)))
ut.assertEqual(len(x), len(y))
for ix, iy in zip(x, y):
assert_equal(ix, iy)
elif isinstance(y, dict):
assert(issubclass(y.__class__, dict))
# ut.assertEqual(type(x), type(y))
ut.assertEqual(len(x), len(y))
for k, iy in y.items():
ix = x[k]
assert_equal(ix, iy)
elif isinstance(y, (np.ndarray, pd.DataFrame)):
if units.has_units(y) and (not units.has_units(x)): # pragma: debug
y = units.get_data(y)
elif (not units.has_units(y)) and units.has_units(x):
x = units.get_data(x)
np.testing.assert_array_equal(x, y)
else:
if units.has_units(y) and units.has_units(x):
x = units.convert_to(x, units.get_units(y))
assert_equal(units.get_data(x), units.get_data(y))
else:
if units.has_units(y) and (not units.has_units(x)): # pragma: debug
y = units.get_data(y)
elif (not units.has_units(y)) and units.has_units(x):
x = units.get_data(x)
ut.assertEqual(x, y)
def encode(cls, instance, typedef=None):
r"""Encoder for the 'units' scalar property."""
out = units.get_units(instance)
if (not out) and (typedef is not None):
out = typedef
return out
def model_wrapper(cls,
name,
synonyms,
interpolation,
aggregation,
additional_variables,
env=None):
r"""Model wrapper."""
from yggdrasil.languages.Python.YggInterface import YggTimesyncServer
if env is not None:
os.environ.update(env)
rpc = YggTimesyncServer(name)
threads = {}
times = []
tables = {}
table_units = {'base': {}}
table_lock = multitasking.RLock()
default_agg = _default_agg
if not isinstance(aggregation, dict):
default_agg = aggregation
aggregation = {}
while True:
# Check for errors on response threads
for v in threads.values():
if v.check_flag_attr('error_flag'): # pragma: debug
for v in threads.values():
if v.is_alive():
v.terminate()
raise Exception("Error on response thread.")
# Receive values from client models
flag, values, request_id = rpc.recv_from(timeout=1.0)
if not flag:
print("timesync server: End of input.")
break
if len(values) == 0:
rpc.sleep()
continue
t, state = values[:]
t_pd = units.convert_to_pandas_timedelta(t)
client_model = rpc.ocomm[request_id].client_model
# Remove variables marked as external so they are not merged
external_variables = additional_variables.get(client_model, [])
for k in external_variables:
state.pop(k, None)
internal_variables = list(state.keys())
# Update record
with table_lock:
if client_model not in tables:
tables[client_model] = pd.DataFrame({'time': times})
# Update units & aggregation methods
if client_model not in table_units:
# NOTE: this assumes that units will not change
# between timesteps for a single model. Is there a
# case where this might not be true?
table_units[client_model] = {
k: units.get_units(v)
for k, v in state.items()
}
table_units[client_model]['time'] = units.get_units(t)
alt_vars = []
for k, v in synonyms.get(client_model, {}).items():
alt_vars += v['alt']
if v['alt2base'] is not None:
table_units[client_model][k] = units.get_units(
v['alt2base'](*[state[a] for a in v['alt']]))
else:
table_units[client_model][k] = table_units[
client_model][v['alt'][0]]
for k, v in table_units[client_model].items():
table_units['base'].setdefault(k, v)
for k in list(set(state.keys()) - set(alt_vars)):
aggregation.setdefault(k, default_agg)
# Update the state
if t_pd not in times:
times.append(t_pd)
for model, table in tables.items():
new_data = {'time': [t_pd]}
if model == client_model:
new_data.update(
{k: [units.get_data(v)]
for k, v in state.items()})
new_data = pd.DataFrame(new_data)
idx = table['time'].isin([t_pd])
if not idx.any():
table = table.append(new_data, sort=False)
elif model == client_model:
table = table.drop(table.index[idx])
table = table.append(new_data, sort=False)
tables[model] = table.sort_values('time')
# Assign thread to handle checking when data is filled in
threads[request_id] = multitasking.YggTaskLoop(
target=cls.response_loop,
args=(client_model, request_id, rpc, t_pd, internal_variables,
external_variables, tables, table_units, table_lock,
synonyms, interpolation, aggregation))
threads[request_id].start()
# Cleanup threads (only called if there is an error since the
# loop will only be broken when all of the clients have signed
# off, implying that all requests have been responded to).
for v in threads.values():
if v.is_alive(): # pragma: debug
v.wait(0.5)
for v in threads.values():
if v.is_alive(): # pragma: debug
v.terminate()
def test_get_units(self, vars_units, vars_nounits):
r"""Test get_units."""
for v in vars_nounits:
assert (units.get_units(v) == '')
for v in vars_units:
assert (units.get_units(v) == str(units.as_unit('cm').units))
