def _init_before_open(self, **kwargs): r"""Set null connection and channel.""" if not hasattr(self, 'rmq_lock'): self.rmq_lock = multitasking.RLock() self.connection = None self.channel = None self._opening = multitasking.ProcessEvent() self._closing = multitasking.ProcessEvent() self._server_class = RMQServer self._server_kwargs = {'comm_cls': self.__class__} super(RMQComm, self)._init_before_open(**kwargs)
def _init_before_open(self, **kwargs): r"""Initialize null variables and RMQ async thread.""" self.times_connected = 0 self.rmq_thread_count = 0 self.rmq_thread = self.new_run_thread() self._opening = False self._closing = False self._reconnecting = False self._close_called = False self._buffered_messages = [] self._qres = None self._qres_lock = multitasking.RLock() self._qres_event = multitasking.Event() self._qres_event.set() super(RMQAsyncComm, self)._init_before_open(**kwargs)
def model_wrapper(cls, name, synonyms, interpolation, aggregation, additional_variables, env=None): r"""Model wrapper.""" from yggdrasil.languages.Python.YggInterface import YggTimesyncServer if env is not None: os.environ.update(env) rpc = YggTimesyncServer(name) threads = {} times = [] tables = {} table_units = {'base': {}} table_lock = multitasking.RLock() default_agg = _default_agg if not isinstance(aggregation, dict): default_agg = aggregation aggregation = {} while True: # Check for errors on response threads for v in threads.values(): if v.check_flag_attr('error_flag'): # pragma: debug for v in threads.values(): if v.is_alive(): v.terminate() raise Exception("Error on response thread.") # Receive values from client models flag, values, request_id = rpc.recv_from(timeout=1.0) if not flag: print("timesync server: End of input.") break if len(values) == 0: rpc.sleep() continue t, state = values[:] t_pd = units.convert_to_pandas_timedelta(t) client_model = rpc.ocomm[request_id].client_model # Remove variables marked as external so they are not merged external_variables = additional_variables.get(client_model, []) for k in external_variables: state.pop(k, None) internal_variables = list(state.keys()) # Update record with table_lock: if client_model not in tables: tables[client_model] = pd.DataFrame({'time': times}) # Update units & aggregation methods if client_model not in table_units: # NOTE: this assumes that units will not change # between timesteps for a single model. Is there a # case where this might not be true? table_units[client_model] = { k: units.get_units(v) for k, v in state.items() } table_units[client_model]['time'] = units.get_units(t) alt_vars = [] for k, v in synonyms.get(client_model, {}).items(): alt_vars += v['alt'] if v['alt2base'] is not None: table_units[client_model][k] = units.get_units( v['alt2base'](*[state[a] for a in v['alt']])) else: table_units[client_model][k] = table_units[ client_model][v['alt'][0]] for k, v in table_units[client_model].items(): table_units['base'].setdefault(k, v) for k in list(set(state.keys()) - set(alt_vars)): aggregation.setdefault(k, default_agg) # Update the state if t_pd not in times: times.append(t_pd) for model, table in tables.items(): new_data = {'time': [t_pd]} if model == client_model: new_data.update( {k: [units.get_data(v)] for k, v in state.items()}) new_data = pd.DataFrame(new_data) idx = table['time'].isin([t_pd]) if not idx.any(): table = table.append(new_data, sort=False) elif model == client_model: table = table.drop(table.index[idx]) table = table.append(new_data, sort=False) tables[model] = table.sort_values('time') # Assign thread to handle checking when data is filled in threads[request_id] = multitasking.YggTaskLoop( target=cls.response_loop, args=(client_model, request_id, rpc, t_pd, internal_variables, external_variables, tables, table_units, table_lock, synonyms, interpolation, aggregation)) threads[request_id].start() # Cleanup threads (only called if there is an error since the # loop will only be broken when all of the clients have signed # off, implying that all requests have been responded to). for v in threads.values(): if v.is_alive(): # pragma: debug v.wait(0.5) for v in threads.values(): if v.is_alive(): # pragma: debug v.terminate()
def _init_before_open(self, context=None, socket_type=None, socket_action=None, topic_filter='', dealer_identity=None, new_process=False, reply_socket_address=None, **kwargs): r"""Initialize defaults for socket type/action based on direction.""" self.reply_socket_lock = multitasking.RLock() self.socket_lock = multitasking.RLock() # Client/Server things if self.is_client: socket_type = 'DEALER' socket_action = 'connect' self.direction = 'send' if self.is_server: socket_type = 'DEALER' socket_action = 'connect' self.direction = 'recv' # Set defaults if socket_type is None: if self.direction == 'recv': socket_type = _socket_recv_types[_default_socket_type] elif self.direction == 'send': socket_type = _socket_send_types[_default_socket_type] if not (self.is_client or self.is_server): if socket_type in ['PULL', 'SUB', 'REP', 'DEALER']: self.direction = 'recv' elif socket_type in ['PUSH', 'PUB', 'REQ', 'ROUTER']: self.direction = 'send' if socket_action is None: if self.port in ['inproc', 'ipc']: if socket_type in ['PULL', 'SUB', 'REQ', 'DEALER']: socket_action = 'connect' elif socket_type in ['PUSH', 'PUB', 'REP', 'ROUTER']: socket_action = 'bind' else: if self.direction == 'recv': socket_action = 'connect' elif self.direction == 'send': socket_action = 'bind' elif self.port is None: socket_action = 'bind' else: socket_action = 'connect' if new_process: self.context = zmq.Context() else: self.context = context or _global_context self.socket_type_name = socket_type self.socket_type = getattr(zmq, socket_type) self.socket_action = socket_action self.socket = self.context.socket(self.socket_type) self.socket.setsockopt(zmq.LINGER, 0) self.topic_filter = tools.str2bytes(topic_filter) if dealer_identity is None: dealer_identity = str(uuid.uuid4()) self.dealer_identity = tools.str2bytes(dealer_identity) self._openned = False self._bound = False self._connected = False self._recv_identities = set([]) # Reply socket attributes self.zmq_sleeptime = int(10000 * self.sleeptime) self.reply_socket_address = reply_socket_address self.reply_socket_send = None self.reply_socket_recv = {} self._n_zmq_sent = 0 self._n_zmq_recv = {} self._n_reply_sent = 0 self._n_reply_recv = {} self._server_class = ZMQProxy self._server_kwargs = dict(zmq_context=self.context, nretry=4, retry_timeout=2.0 * self.sleeptime) super(ZMQComm, self)._init_before_open(**kwargs)