def test_get_units(self): r"""Test get_units.""" for v in self._vars_nounits: self.assert_equal(units.get_units(v), '') for v in self._vars_units: self.assert_equal(units.get_units(v), str(units.as_unit('cm').units))
def assertUnitsEqual(self, first, second, msg=None): r"""Assertion for equality in case of objects with units.""" if units.has_units(first) and units.has_units(second): first = units.convert_to(first, units.get_units(second)) self.assertEqual(units.get_data(first), units.get_data(second), msg=msg)
def concatenate(cls, objects, as_array=False, **kwargs): r"""Concatenate objects to get object that would be recieved if the concatenated serialization were deserialized. Args: objects (list): Objects to be concatenated. as_array (bool, optional): If True, the objects in the list are complete columns in a table and as_format is set to True. Defaults to False. **kwargs: Additional keyword arguments are ignored. Returns: list: Set of objects that results from concatenating those provided. """ if len(objects) == 0: return [] if as_array: units_list = [units.get_units(ix) for ix in objects[0]] out = [[ units.add_units(np.hstack([x[i] for x in objects]), u) for i, u in enumerate(units_list) ]] elif isinstance(objects[0], bytes): out = [b''.join(objects)] else: return super(DefaultSerialize, cls).concatenate(objects, **kwargs) return out
def _file_send(self, msg): assert (isinstance(msg, dict)) for k, v in msg.items(): if isinstance(v, np.ndarray): dims = [] v = self.transform_type_send(v) for i, d in enumerate(v.shape): if i == 0: idim = k else: idim = '%s%d' % (k, i) dims.append(idim) self._fd_netcdf.createDimension(idim, d) var = self._fd_netcdf.createVariable(k, v.dtype, dims) var[:] = v if units.has_units(v): var.units = units.get_units(v) else: # pragma: debug raise TypeError("Type '%s' no supported." % type(msg)) self._last_size = self.file_size self.fd.seek(self._last_size)
def assert_equal(x, y): r"""Assert that two messages are equivalent. Args: x (object): Python object to compare against y. y (object): Python object to compare against x. Raises: AssertionError: If the two messages are not equivalent. """ if isinstance(y, (list, tuple)): assert(isinstance(x, (list, tuple))) ut.assertEqual(len(x), len(y)) for ix, iy in zip(x, y): assert_equal(ix, iy) elif isinstance(y, dict): assert(issubclass(y.__class__, dict)) # ut.assertEqual(type(x), type(y)) ut.assertEqual(len(x), len(y)) for k, iy in y.items(): ix = x[k] assert_equal(ix, iy) elif isinstance(y, (np.ndarray, pd.DataFrame)): if units.has_units(y) and (not units.has_units(x)): # pragma: debug y = units.get_data(y) elif (not units.has_units(y)) and units.has_units(x): x = units.get_data(x) np.testing.assert_array_equal(x, y) else: if units.has_units(y) and units.has_units(x): x = units.convert_to(x, units.get_units(y)) assert_equal(units.get_data(x), units.get_data(y)) else: if units.has_units(y) and (not units.has_units(x)): # pragma: debug y = units.get_data(y) elif (not units.has_units(y)) and units.has_units(x): x = units.get_data(x) ut.assertEqual(x, y)
def encode(cls, instance, typedef=None): r"""Encoder for the 'units' scalar property.""" out = units.get_units(instance) if (not out) and (typedef is not None): out = typedef return out
def model_wrapper(cls, name, synonyms, interpolation, aggregation, additional_variables, env=None): r"""Model wrapper.""" from yggdrasil.languages.Python.YggInterface import YggTimesyncServer if env is not None: os.environ.update(env) rpc = YggTimesyncServer(name) threads = {} times = [] tables = {} table_units = {'base': {}} table_lock = multitasking.RLock() default_agg = _default_agg if not isinstance(aggregation, dict): default_agg = aggregation aggregation = {} while True: # Check for errors on response threads for v in threads.values(): if v.check_flag_attr('error_flag'): # pragma: debug for v in threads.values(): if v.is_alive(): v.terminate() raise Exception("Error on response thread.") # Receive values from client models flag, values, request_id = rpc.recv_from(timeout=1.0) if not flag: print("timesync server: End of input.") break if len(values) == 0: rpc.sleep() continue t, state = values[:] t_pd = units.convert_to_pandas_timedelta(t) client_model = rpc.ocomm[request_id].client_model # Remove variables marked as external so they are not merged external_variables = additional_variables.get(client_model, []) for k in external_variables: state.pop(k, None) internal_variables = list(state.keys()) # Update record with table_lock: if client_model not in tables: tables[client_model] = pd.DataFrame({'time': times}) # Update units & aggregation methods if client_model not in table_units: # NOTE: this assumes that units will not change # between timesteps for a single model. Is there a # case where this might not be true? table_units[client_model] = { k: units.get_units(v) for k, v in state.items() } table_units[client_model]['time'] = units.get_units(t) alt_vars = [] for k, v in synonyms.get(client_model, {}).items(): alt_vars += v['alt'] if v['alt2base'] is not None: table_units[client_model][k] = units.get_units( v['alt2base'](*[state[a] for a in v['alt']])) else: table_units[client_model][k] = table_units[ client_model][v['alt'][0]] for k, v in table_units[client_model].items(): table_units['base'].setdefault(k, v) for k in list(set(state.keys()) - set(alt_vars)): aggregation.setdefault(k, default_agg) # Update the state if t_pd not in times: times.append(t_pd) for model, table in tables.items(): new_data = {'time': [t_pd]} if model == client_model: new_data.update( {k: [units.get_data(v)] for k, v in state.items()}) new_data = pd.DataFrame(new_data) idx = table['time'].isin([t_pd]) if not idx.any(): table = table.append(new_data, sort=False) elif model == client_model: table = table.drop(table.index[idx]) table = table.append(new_data, sort=False) tables[model] = table.sort_values('time') # Assign thread to handle checking when data is filled in threads[request_id] = multitasking.YggTaskLoop( target=cls.response_loop, args=(client_model, request_id, rpc, t_pd, internal_variables, external_variables, tables, table_units, table_lock, synonyms, interpolation, aggregation)) threads[request_id].start() # Cleanup threads (only called if there is an error since the # loop will only be broken when all of the clients have signed # off, implying that all requests have been responded to). for v in threads.values(): if v.is_alive(): # pragma: debug v.wait(0.5) for v in threads.values(): if v.is_alive(): # pragma: debug v.terminate()
def test_get_units(self, vars_units, vars_nounits): r"""Test get_units.""" for v in vars_nounits: assert (units.get_units(v) == '') for v in vars_units: assert (units.get_units(v) == str(units.as_unit('cm').units))