def assertUnitsEqual(self, first, second, msg=None): r"""Assertion for equality in case of objects with units.""" if units.has_units(first) and units.has_units(second): first = units.convert_to(first, units.get_units(second)) self.assertEqual(units.get_data(first), units.get_data(second), msg=msg)
def run(tmin, tmax, tstep): mass = 2000.0 mesh_in = YggInput('mesh') mesh_out = YggOutput('mesh') mass = units.add_units(mass, 'g') tmin = units.add_units(tmin, 'hrs') tmax = units.add_units(tmax, 'hrs') tstep = units.add_units(tstep, 'hrs') t = tmin while t <= tmax: # Receive mesh as input flag, mesh = mesh_in.recv() if not flag: raise Exception("Error receiving mesh from input") mesh = mesh.as_trimesh() # Grow mesh # (pretend this is a biologically complex calculation) scale = units.get_data(mass * t / units.add_units(1.5e6, 'g*hrs')) mesh.vertices[:, 2] += mesh.vertices[:, 2] * scale # Send mesh to output for this timestep flag = mesh_out.send(mesh) if not flag: raise Exception("Error sending mesh to output") # Advance time step t += tstep return mesh
def __call__(self, x): r"""Call filter on the provided message. Args: x (object): Message object to filter. Returns: bool: True if the message will pass through the filter, False otherwise. """ try: out = self.evaluate_filter(x) except ValueError: if backwards.PY2 and units.has_units(x): out = self.evaluate_filter(units.get_data(x)) else: raise # pragma: debug if isinstance(out, np.ndarray): assert (out.dtype == bool) out = bool(out.all()) elif isinstance(out, np.bool_): out = bool(out) try: assert (isinstance(out, bool)) except AssertionError: # pragma: debug print(out, type(out)) raise return out
def validate(cls, obj, raise_errors=False): r"""Validate an object to check if it could be of this type. Args: obj (object): Object to validate. raise_errors (bool, optional): If True, errors will be raised when the object fails to be validated. Defaults to False. Returns: bool: True if the object could be of this type, False otherwise. """ if isinstance(obj, np.ndarray) and (obj.ndim == 0): obj = obj.reshape((1, ))[0] if super(ScalarMetaschemaType, cls).validate(units.get_data(obj), raise_errors=raise_errors): dtype = ScalarMetaschemaProperties.data2dtype(obj) if cls.is_fixed and ('subtype' in cls.fixed_properties): type_list = [ ScalarMetaschemaProperties._valid_types[ cls.fixed_properties['subtype']] ] else: type_list = ScalarMetaschemaProperties._valid_numpy_types if dtype.name.startswith(tuple(type_list)): return True else: if raise_errors: raise ValueError( ("dtype %s dosn't corresponding with any " + "of the accepted types: %s") % (str(dtype), str(type_list))) return False
def format_message(args, fmt_str): r"""Format a message from a list of arguments and a format string. Args: args (list, obj): List of arguments or single argument that should be formatted using the format string. fmt_str (str, bytes): Format string that should be used to format the arguments. Returns: str, bytes: Formatted message. The type will match the type of the fmt_str. Raises: RuntimeError: If the number of arguments does not match the number of format fields. """ if not isinstance(args, (tuple, list)): args = (args, ) nfmt = len(extract_formats(fmt_str)) args_ = [] if len(args) < nfmt: raise RuntimeError("Number of arguments (%d) does not match " % len(args) + "number of format fields (%d)." % nfmt) for a0 in args: a = units.get_data(a0) if np.iscomplexobj(a): args_ += [a.real, a.imag] else: args_.append(a) out = backwards.format_bytes(fmt_str, tuple(args_)) return out
def run(mesh, tmin, tmax, tstep): mass = 2000.0 light_rpc = YggRpcClient('light_plant') light_out = YggOutput('light') plant2root = YggTimesync('plant2root') mass = units.add_units(mass, 'g') tmin = units.add_units(tmin, 'hrs') tmax = units.add_units(tmax, 'hrs') tstep = units.add_units(tstep, 'hrs') t = tmin i = 0 while t <= tmax: # Perform send portion of call to synchronize data for time step # with the root model root_state = {'mass': mass} flag = plant2root.send(t, root_state) if not flag: raise Exception("Error performing time-step synchronization " "with root model.") # Get light data by calling light model flag, light = light_rpc.call(mesh.vertices[:, 2], t) if not flag: raise Exception("Error calling light model") # Perform receive portion of call to synchronize data for time step # with the root model flag, root_state = plant2root.recv() if not flag: raise Exception("Error performing recv for time-step " "synchronization with root model.") mass = root_state['mass'] # Grow mesh # (pretend this is a biologically complex calculation) scale = units.get_data(mass * light / units.add_units(1.0, 'kg')) mesh.vertices[:, 2] += mesh.vertices[:, 2] * scale # Save mesh for this timestep filename_mesh = os.path.join(_dir, f'../output/mesh_{i:03d}.obj') with open(filename_mesh, 'w') as fd: mesh.export(fd, 'obj') # Send light to output flag = light_out.send(light) if not flag: raise Exception("Error sending light to output") # Advance time step t += tstep i += 1 return mesh
def get_testing_options(cls, not_as_frames=False, no_names=False, **kwargs): r"""Method to return a dictionary of testing options for this class. Args: not_as_frames (bool, optional): If True, the returned example includes data that is not in a pandas data frame. Defaults to False. no_names (bool, optional): If True, an example is returned where the names are not provided to the deserializer. Defaults to False. Returns: dict: Dictionary of variables to use for testing. """ field_names = None out = super(PandasSerialize, cls).get_testing_options(array_columns=True, **kwargs) for k in ['as_array']: # , 'format_str']: if k in out['kwargs']: del out['kwargs'][k] out['extra_kwargs'] = {} out['empty'] = pandas.DataFrame() if no_names: for x in [out['kwargs'], out]: if 'field_names' in x: del x['field_names'] header_line = b'f0\tf1\tf2\n' else: if 'field_names' in out['kwargs']: field_names = [ backwards.as_str(x) for x in out['kwargs']['field_names'] ] header_line = b'name\tcount\tsize\n' out['contents'] = (header_line + b'one\t1\t1.0\n' + b'two\t2\t2.0\n' + b'three\t3\t3.0\n' + b'one\t1\t1.0\n' + b'two\t2\t2.0\n' + b'three\t3\t3.0\n') out['concatenate'] = [([], [])] if not_as_frames: # Strip units since pandas data frames are not serialized with units out['objects'] = [[units.get_data(ix) for ix in x] for x in out['objects']] else: out['objects'] = [ serialize.list2pandas(x, names=field_names) for x in out['objects'] ] out['kwargs'].update(out['typedef']) return out
def to_array(cls, obj): r"""Get np.array representation of the data. Args: obj (object): Object to get array for. Returns: np.ndarray: Array representation of object. """ obj_nounits = units.get_data(obj) if isinstance(obj_nounits, np.ndarray): arr = obj_nounits else: dtype = ScalarMetaschemaProperties.data2dtype(obj_nounits) arr = np.array([obj_nounits], dtype=dtype) return arr
def data2dtype(data): r"""Get numpy data type for an object. Args: data (object): Python object. Returns: np.dtype: Numpy data type. """ data_nounits = units.get_data(data) if isinstance(data_nounits, np.ndarray): dtype = data_nounits.dtype elif isinstance(data_nounits, (list, dict, tuple)): raise MetaschemaTypeError elif isinstance(data_nounits, np.dtype(_valid_types['bytes']).type): dtype = np.array(data_nounits).dtype else: dtype = np.array([data_nounits]).dtype return dtype
def validate(cls, obj, raise_errors=False): r"""Validate an object to check if it could be of this type. Args: obj (object): Object to validate. raise_errors (bool, optional): If True, errors will be raised when the object fails to be validated. Defaults to False. Returns: bool: True if the object could be of this type, False otherwise. """ if not super(OneDArrayMetaschemaType, cls).validate( obj, raise_errors=raise_errors): return False if units.get_data(obj).ndim != 1: if raise_errors: raise ValueError("The array has more than one dimension.") return False return True
def run(mesh, tmin, tmax, tstep): mass = 2000.0 light_rpc = YggRpcClient('light_plant') light_out = YggOutput('light') mass = units.add_units(mass, 'g') tmin = units.add_units(tmin, 'hrs') tmax = units.add_units(tmax, 'hrs') tstep = units.add_units(tstep, 'hrs') t = tmin i = 0 while t <= tmax: # Get light data by calling light model flag, light = light_rpc.call(mesh.vertices[:, 2], t) if not flag: raise Exception("Error calling light model") # Grow mesh # (pretend this is a biologically complex calculation) scale = units.get_data(mass * light / units.add_units(1.0, 'kg')) mesh.vertices[:, 2] += mesh.vertices[:, 2] * scale # Save mesh for this timestep filename_mesh = os.path.join(_dir, f'../output/mesh_{i:03d}.obj') with open(filename_mesh, 'w') as fd: mesh.export(fd, 'obj') # Send light to output flag = light_out.send(light) if not flag: raise Exception("Error sending light to output") # Advance time step t += tstep i += 1 return mesh
def assert_equal(x, y): r"""Assert that two messages are equivalent. Args: x (object): Python object to compare against y. y (object): Python object to compare against x. Raises: AssertionError: If the two messages are not equivalent. """ if isinstance(y, (list, tuple)): assert(isinstance(x, (list, tuple))) ut.assertEqual(len(x), len(y)) for ix, iy in zip(x, y): assert_equal(ix, iy) elif isinstance(y, dict): assert(issubclass(y.__class__, dict)) # ut.assertEqual(type(x), type(y)) ut.assertEqual(len(x), len(y)) for k, iy in y.items(): ix = x[k] assert_equal(ix, iy) elif isinstance(y, (np.ndarray, pd.DataFrame)): if units.has_units(y) and (not units.has_units(x)): # pragma: debug y = units.get_data(y) elif (not units.has_units(y)) and units.has_units(x): x = units.get_data(x) np.testing.assert_array_equal(x, y) else: if units.has_units(y) and units.has_units(x): x = units.convert_to(x, units.get_units(y)) assert_equal(units.get_data(x), units.get_data(y)) else: if units.has_units(y) and (not units.has_units(x)): # pragma: debug y = units.get_data(y) elif (not units.has_units(y)) and units.has_units(x): x = units.get_data(x) ut.assertEqual(x, y)
def test_get_data(self): r"""Test get_data.""" for v in self._vars_nounits: self.assert_equal(units.get_data(v), v) for vno, v in zip(self._vars_nounits, self._vars_units): self.assert_equal(units.get_data(v), np.array(vno))
def get_testing_options(cls, not_as_frames=False, no_names=False, no_header=False, **kwargs): r"""Method to return a dictionary of testing options for this class. Args: not_as_frames (bool, optional): If True, the returned example includes data that is not in a pandas data frame. Defaults to False. no_names (bool, optional): If True, an example is returned where the names are not provided to the deserializer. Defaults to False. no_header (bool, optional): If True, an example is returned where a header is not included. Defaults to False. Returns: dict: Dictionary of variables to use for testing. """ kwargs.setdefault('table_string_type', 'string') field_names = None out = super(PandasSerialize, cls).get_testing_options(array_columns=True, **kwargs) if kwargs['table_string_type'] == 'bytes': out['kwargs']['str_as_bytes'] = True for k in ['as_array']: # , 'format_str']: if k in out['kwargs']: del out['kwargs'][k] out['extra_kwargs'] = {} if no_names: for x in [out['kwargs'], out]: if 'field_names' in x: del x['field_names'] header_line = b'f0\tf1\tf2\n' elif no_header: for x in [out['kwargs'], out]: if 'field_names' in x: del x['field_names'] header_line = b'' out['kwargs']['no_header'] = True for x in out['typedef']['items']: x.pop('title', None) else: if 'field_names' in out['kwargs']: field_names = out['kwargs']['field_names'] header_line = b'name\tcount\tsize\n' out['contents'] = (header_line + b'one\t1\t1.0\n' + b'two\t2\t2.0\n' + b'three\t3\t3.0\n' + b'one\t1\t1.0\n' + b'two\t2\t2.0\n' + b'three\t3\t3.0\n') out['concatenate'] = [([], [])] if not_as_frames: # Strip units since pandas data frames are not serialized with units out['objects'] = [[units.get_data(ix) for ix in x] for x in out['objects']] elif no_header: out['objects'] = [serialize.list2pandas(x) for x in out['objects']] out['dtype'] = np.dtype(','.join( [x[1] for x in out['dtype'].descr])) else: if field_names is None: field_names = ['f0', 'f1', 'f2'] out['objects'] = [ serialize.list2pandas(x, names=field_names) for x in out['objects'] ] out['kwargs']['datatype'] = copy.deepcopy(out['typedef']) if no_names: for x in out['kwargs']['datatype']['items']: x.pop('title', None) out['empty'] = pandas.DataFrame(np.zeros(0, out['dtype'])) return out
def fcond(x): return (units.get_data(x) != 3)
def model_wrapper(cls, name, synonyms, interpolation, aggregation, additional_variables, env=None): r"""Model wrapper.""" from yggdrasil.languages.Python.YggInterface import YggTimesyncServer if env is not None: os.environ.update(env) rpc = YggTimesyncServer(name) threads = {} times = [] tables = {} table_units = {'base': {}} table_lock = multitasking.RLock() default_agg = _default_agg if not isinstance(aggregation, dict): default_agg = aggregation aggregation = {} while True: # Check for errors on response threads for v in threads.values(): if v.check_flag_attr('error_flag'): # pragma: debug for v in threads.values(): if v.is_alive(): v.terminate() raise Exception("Error on response thread.") # Receive values from client models flag, values, request_id = rpc.recv_from(timeout=1.0) if not flag: print("timesync server: End of input.") break if len(values) == 0: rpc.sleep() continue t, state = values[:] t_pd = units.convert_to_pandas_timedelta(t) client_model = rpc.ocomm[request_id].client_model # Remove variables marked as external so they are not merged external_variables = additional_variables.get(client_model, []) for k in external_variables: state.pop(k, None) internal_variables = list(state.keys()) # Update record with table_lock: if client_model not in tables: tables[client_model] = pd.DataFrame({'time': times}) # Update units & aggregation methods if client_model not in table_units: # NOTE: this assumes that units will not change # between timesteps for a single model. Is there a # case where this might not be true? table_units[client_model] = { k: units.get_units(v) for k, v in state.items() } table_units[client_model]['time'] = units.get_units(t) alt_vars = [] for k, v in synonyms.get(client_model, {}).items(): alt_vars += v['alt'] if v['alt2base'] is not None: table_units[client_model][k] = units.get_units( v['alt2base'](*[state[a] for a in v['alt']])) else: table_units[client_model][k] = table_units[ client_model][v['alt'][0]] for k, v in table_units[client_model].items(): table_units['base'].setdefault(k, v) for k in list(set(state.keys()) - set(alt_vars)): aggregation.setdefault(k, default_agg) # Update the state if t_pd not in times: times.append(t_pd) for model, table in tables.items(): new_data = {'time': [t_pd]} if model == client_model: new_data.update( {k: [units.get_data(v)] for k, v in state.items()}) new_data = pd.DataFrame(new_data) idx = table['time'].isin([t_pd]) if not idx.any(): table = table.append(new_data, sort=False) elif model == client_model: table = table.drop(table.index[idx]) table = table.append(new_data, sort=False) tables[model] = table.sort_values('time') # Assign thread to handle checking when data is filled in threads[request_id] = multitasking.YggTaskLoop( target=cls.response_loop, args=(client_model, request_id, rpc, t_pd, internal_variables, external_variables, tables, table_units, table_lock, synonyms, interpolation, aggregation)) threads[request_id].start() # Cleanup threads (only called if there is an error since the # loop will only be broken when all of the clients have signed # off, implying that all requests have been responded to). for v in threads.values(): if v.is_alive(): # pragma: debug v.wait(0.5) for v in threads.values(): if v.is_alive(): # pragma: debug v.terminate()
def test_get_data(self, vars_units, vars_nounits, nested_approx): r"""Test get_data.""" for v in vars_nounits: assert (units.get_data(v) == nested_approx(v)) for vno, v in zip(vars_nounits, vars_units): assert (units.get_data(v) == nested_approx(vno))