def setup(self): r"""Start driver and instance.""" if self.direction is None: # pragma: debug return nprev_comm = self.comm_count nprev_thread = self.thread_count nprev_fd = self.fd_count # File if self.is_file and (self.direction == 'input'): with open(self.filename, 'wb') as fd: fd.write(self.testing_options['contents']) assert(os.path.isfile(self.filename)) # Drivers args, kwargs = self.iodriver_args self.iodriver = self.iodriver_class(*args, **kwargs) self.iodriver.start() self.iodriver.wait_for_loop() # Test comm if not self.is_file: if self.direction == 'input': kws = self.iodriver.icomm.opp_comm_kwargs() kws.update(self.test_comm_kwargs) self.test_comm = get_comm('in', **kws) elif self.direction == 'output': kws = self.iodriver.ocomm.opp_comm_kwargs() kws.update(self.test_comm_kwargs) self.test_comm = get_comm('out', **kws) # Test class super(TestBase, self).setup(nprev_comm=nprev_comm, nprev_thread=nprev_thread, nprev_fd=nprev_fd)
def __init__(self, name, comm=None, **kwargs): self.comm_list = [] self.curr_comm_index = 0 self.eof_recv = [] address = kwargs.pop('address', None) if (comm in [None, 'ForkComm']): if isinstance(address, list): ncomm = len(address) else: ncomm = 0 comm = [None for i in range(ncomm)] assert (isinstance(comm, list)) ncomm = len(comm) for i in range(ncomm): if comm[i] is None: comm[i] = {} if comm[i].get('name', None) is None: comm[i]['name'] = get_comm_name(name, i) if isinstance(address, list): assert (len(address) == ncomm) for i in range(ncomm): comm[i]['address'] = address[i] for i in range(ncomm): ikw = dict(**kwargs) ikw.update(**comm[i]) iname = ikw.pop('name') self.comm_list.append(get_comm(iname, **ikw)) self.eof_recv.append(0) if ncomm > 0: kwargs['address'] = [x.address for x in self.comm_list] kwargs['comm'] = 'ForkComm' super(ForkComm, self).__init__(name, **kwargs) assert (not self.single_use) assert (not self.is_server) assert (not self.is_client)
def __init__(self, name, request_comm=None, response_kwargs=None, dont_open=False, **kwargs): if response_kwargs is None: response_kwargs = dict() icomm_name = name icomm_kwargs = kwargs icomm_kwargs['direction'] = 'recv' icomm_kwargs['dont_open'] = True icomm_kwargs['comm'] = request_comm self.response_kwargs = response_kwargs self.icomm = get_comm(icomm_name, **icomm_kwargs) self.ocomm = OrderedDict() self.response_kwargs.setdefault('comm', self.icomm.comm_class) self.response_kwargs.setdefault('recv_timeout', self.icomm.recv_timeout) self.response_kwargs.setdefault('language', self.icomm.language) self._used_response_comms = dict() self.clients = [] self.closed_clients = [] self.nclients_expected = int(os.environ.get('YGG_NCLIENTS', 0)) super(ServerComm, self).__init__(self.icomm.name, dont_open=dont_open, recv_timeout=self.icomm.recv_timeout, is_interface=self.icomm.is_interface, direction='recv', no_suffix=True, address=self.icomm.address)
def __init__(self, name, request_comm=None, response_kwargs=None, dont_open=False, **kwargs): if response_kwargs is None: response_kwargs = dict() ocomm_name = name ocomm_kwargs = kwargs ocomm_kwargs['direction'] = 'send' ocomm_kwargs['dont_open'] = True ocomm_kwargs['comm'] = request_comm self.response_kwargs = response_kwargs self.ocomm = get_comm(ocomm_name, **ocomm_kwargs) self.icomm = dict() self.icomm_order = [] self.response_kwargs.setdefault('comm', self.ocomm.comm_class) self.response_kwargs.setdefault('recv_timeout', self.ocomm.recv_timeout) self.response_kwargs.setdefault('language', self.ocomm.language) super(ClientComm, self).__init__(self.ocomm.name, dont_open=dont_open, recv_timeout=self.ocomm.recv_timeout, is_interface=self.ocomm.is_interface, direction='send', no_suffix=True, address=self.ocomm.address)
def create_response_comm(self, header): r"""Create a response comm based on information from the last header.""" if not isinstance(header, dict): # pragma: debug raise RuntimeError("No header received with last message.") elif 'response_address' not in header: # pragma: debug raise RuntimeError( "Last header does not contain response address.") comm_kwargs = dict(address=header['response_address'], direction='send', **self.response_kwargs) if self.direct_connection: comm_kwargs['is_response_client'] = True else: comm_kwargs['is_response_server'] = True response_id = header['request_id'] while response_id in self.requests: # pragma: debug response_id += str(uuid.uuid4()) header['response_id'] = response_id if header['response_address'] not in self.ocomm: self.ocomm[header['response_address']] = get_comm( self.name + '.server_response_comm.' + response_id, **comm_kwargs) client_model = header.get('model', '') self.ocomm[header['response_address']].client_model = client_model if client_model and (client_model not in self.clients): self.clients.append(client_model) self.requests[response_id] = Request(header['response_address'], header['request_id'])
def rank_comm(self, name, mpi_rank, rank_kwargs, close_comm): r"""MPI communicator for rank > 0 processes.""" if mpi_rank > 0: x = get_comm(name, **rank_kwargs) yield x close_comm(x) else: yield None
def test_comm(self, iodriver, direction, filecomm, test_comm_kwargs, close_comm): r"""Communicator for testing.""" test_comm = None if not filecomm: if direction == 'input': kws = iodriver.icomm.opp_comm_kwargs() kws.update(test_comm_kwargs) test_comm = get_comm('in', **kws) elif direction == 'output': kws = iodriver.ocomm.opp_comm_kwargs() kws.update(test_comm_kwargs) test_comm = get_comm('out', **kws) try: yield test_comm finally: if test_comm is not None: close_comm(test_comm)
def create_recv_comm(self, name, commtype, send_comm, testing_options, **kwargs): r"""Create a receive communicator.""" kws = self.get_recv_comm_kwargs(commtype, send_comm, testing_options, **kwargs) x = get_comm(name, **kws) assert (x.is_open) x.drain_server_signon_messages() return x
def setup(self): r"""Start driver and instance.""" if self.direction is None: # pragma: debug return nprev_comm = self.comm_count nprev_thread = self.thread_count nprev_fd = self.fd_count idriver_class = self.idriver_class odriver_class = self.odriver_class # File if self.is_file and (self.direction == 'input'): with open(self.filename, 'wb') as fd: fd.write(self.testing_options['contents']) # Drivers comm_env = None if idriver_class is not None: args, kwargs = self.idriver_args self.idriver = idriver_class(*args, **kwargs) self.idriver.start() comm_env = self.idriver.comm_env if odriver_class is not None: args, kwargs = self.odriver_args if comm_env is not None: kwargs['comm_env'] = comm_env self.odriver = odriver_class(*args, **kwargs) self.odriver.start() # Test comm if self.direction == 'input': os.environ.update(self.idriver.env) if self.odriver is not None: kws = self.odriver.icomm.opp_comm_kwargs() kws.update(self.test_comm_kwargs) self.test_comm = get_comm('in', **kws) elif self.direction == 'output': os.environ.update(self.odriver.env) if self.idriver is not None: kws = self.idriver.ocomm.opp_comm_kwargs() kws.update(self.test_comm_kwargs) self.test_comm = get_comm('out', **kws) # Test class super(TestBase, self).setup(nprev_comm=nprev_comm, nprev_thread=nprev_thread, nprev_fd=nprev_fd)
def create_response_comm(self): r"""Create a response comm based on information from the last header.""" if not isinstance(self.icomm._last_header, dict): # pragma: debug raise RuntimeError("No header received with last message.") elif 'response_address' not in self.icomm._last_header: # pragma: debug raise RuntimeError("Last header does not contain response address.") comm_kwargs = dict(address=self.icomm._last_header['response_address'], direction='send', is_response_server=True, single_use=True, **self.response_kwargs) self.ocomm = get_comm(self.name + '.server_response_comm', **comm_kwargs)
def __init__(self, name, comm_list=None, is_async=False, **kwargs): child_keys = ['serializer_class', 'serializer_kwargs', # 'datatype', 'format_str', 'field_names', 'field_units', 'as_array'] noprop_keys = ['send_converter', 'recv_converter', 'filter', 'transform'] child_kwargs = {k: kwargs.pop(k) for k in child_keys if k in kwargs} noprop_kwargs = {k: kwargs.pop(k) for k in noprop_keys if k in kwargs} self.comm_list = [] self.curr_comm_index = 0 self.eof_recv = [] address = kwargs.pop('address', None) if comm_list is None: if isinstance(address, list): ncomm = len(address) else: ncomm = 0 comm_list = [None for i in range(ncomm)] assert(isinstance(comm_list, list)) ncomm = len(comm_list) for i in range(ncomm): if comm_list[i] is None: comm_list[i] = {} if comm_list[i].get('name', None) is None: comm_list[i]['name'] = get_comm_name(name, i) for k in child_kwargs.keys(): if k in comm_list[i]: # pragma: debug raise ValueError("The keyword '%s' was specified for both the " "root ForkComm and a child comm, but can only " "be present for one." % k) if isinstance(address, list): assert(len(address) == ncomm) for i in range(ncomm): comm_list[i]['address'] = address[i] for i in range(ncomm): ikw = copy.deepcopy(kwargs) ikw.update(child_kwargs) ikw.update(comm_list[i]) ikw.setdefault('use_async', is_async) iname = ikw.pop('name') self.comm_list.append(get_comm(iname, **ikw)) self.eof_recv.append(0) if ncomm > 0: kwargs['address'] = [x.address for x in self.comm_list] kwargs.update(noprop_kwargs) super(ForkComm, self).__init__(name, is_async=is_async, **kwargs) assert(not self.single_use) assert(not self.is_server) assert(not self.is_client)
def __init__(self, name, request_commtype=None, response_kwargs=None, dont_open=False, is_async=False, direct_connection=False, **kwargs): if response_kwargs is None: response_kwargs = dict() icomm_name = name icomm_kwargs = kwargs icomm_kwargs.update(direction='recv', dont_open=True, commtype=request_commtype) icomm_kwargs.setdefault('is_server', True) icomm_kwargs.setdefault('use_async', is_async) if icomm_kwargs.get('use_async', False): icomm_kwargs.setdefault('async_recv_method', 'recv_message') self.direct_connection = direct_connection self.response_kwargs = response_kwargs self.icomm = get_comm(icomm_name, **icomm_kwargs) self.ocomm = OrderedDict() self.requests = OrderedDict() self.response_kwargs.setdefault('is_interface', self.icomm.is_interface) self.response_kwargs.setdefault('commtype', self.icomm._commtype) self.response_kwargs.setdefault('recv_timeout', self.icomm.recv_timeout) self.response_kwargs.setdefault('language', self.icomm.language) self.response_kwargs.setdefault('use_async', self.icomm.is_async) self.response_kwargs.setdefault('env', self.icomm.env) self.clients = [] self.closed_clients = [] self.nclients_expected = int(os.environ.get('YGG_NCLIENTS', 0)) super(ServerComm, self).__init__(self.icomm.name, dont_open=dont_open, recv_timeout=self.icomm.recv_timeout, is_interface=self.icomm.is_interface, direction='recv', no_suffix=True, address=self.icomm.address, is_async=self.icomm.is_async, env=self.icomm.env)
def __init__(self, name, request_commtype=None, response_kwargs=None, dont_open=False, is_async=False, direct_connection=False, **kwargs): if response_kwargs is None: response_kwargs = dict() ocomm_name = name ocomm_kwargs = kwargs ocomm_kwargs['direction'] = 'send' ocomm_kwargs['dont_open'] = True ocomm_kwargs['commtype'] = request_commtype ocomm_kwargs.setdefault('use_async', is_async) if direct_connection: ocomm_kwargs.setdefault('is_client', True) self.direct_connection = direct_connection self.response_kwargs = response_kwargs self.ocomm = get_comm(ocomm_name, **ocomm_kwargs) self.icomm = None self.request_order = [] self.responses = OrderedDict() for k, v in self.ocomm.get_response_comm_kwargs.items(): self.response_kwargs.setdefault(k, v) self.response_kwargs.setdefault('is_interface', self.ocomm.is_interface) self.response_kwargs.setdefault('recv_timeout', self.ocomm.recv_timeout) self.response_kwargs.setdefault('language', self.ocomm.language) self.response_kwargs.setdefault('use_async', self.ocomm.is_async) self.response_kwargs.setdefault('env', self.ocomm.env) super(ClientComm, self).__init__(self.ocomm.name, dont_open=dont_open, recv_timeout=self.ocomm.recv_timeout, is_interface=self.ocomm.is_interface, direction='send', no_suffix=True, address=self.ocomm.address, is_async=self.ocomm.is_async, env=self.ocomm.env)
def __init__(self, name, request_comm=None, response_kwargs=None, dont_open=False, **kwargs): if response_kwargs is None: response_kwargs = dict() icomm_name = name icomm_kwargs = kwargs icomm_kwargs['direction'] = 'recv' icomm_kwargs['dont_open'] = True icomm_kwargs['comm'] = request_comm self.response_kwargs = response_kwargs self.icomm = get_comm(icomm_name, **icomm_kwargs) self.ocomm = None self.response_kwargs.setdefault('comm', self.icomm.comm_class) self.response_kwargs.setdefault('recv_timeout', self.icomm.recv_timeout) self._used_response_comms = dict() super(ServerComm, self).__init__(self.icomm.name, dont_open=dont_open, recv_timeout=self.icomm.recv_timeout, is_interface=self.icomm.is_interface, direction='recv', no_suffix=True, address=self.icomm.address)
def create_instance(self): r"""Create a new instance of the class.""" inst = get_comm(*self.inst_args, **self.inst_kwargs) assert (isinstance(inst, self.import_cls)) return inst
def __init__(self, name, comm_list=None, is_async=False, pattern=None, **kwargs): child_kwargs = { k: kwargs.pop(k) for k in self.child_keys if k in kwargs } noprop_kwargs = { k: kwargs.pop(k) for k in self.noprop_keys if k in kwargs } self.comm_list_backlog = {} self.comm_list = [] self.curr_comm_index = 0 self.eof_recv = [] self.eof_send = [] if pattern is None: if kwargs.get('direction', 'send') == 'recv': pattern = 'cycle' else: pattern = 'broadcast' self.pattern = pattern if kwargs.get('direction', 'send') == 'recv': assert (self.pattern in ['cycle', 'gather']) else: assert (self.pattern in ['cycle', 'scatter', 'broadcast']) address = kwargs.pop('address', None) if comm_list is None: if isinstance(address, list): ncomm = len(address) else: ncomm = 0 comm_list = [None for i in range(ncomm)] assert (isinstance(comm_list, list)) ncomm = len(comm_list) for i in range(ncomm): if comm_list[i] is None: comm_list[i] = {} if comm_list[i].get('name', None) is None: comm_list[i]['name'] = get_comm_name(name, i) for k in child_kwargs.keys(): if k in comm_list[i]: # pragma: debug raise ValueError( "The keyword '%s' was specified for both the " "root ForkComm and a child comm, but can only " "be present for one." % k) if isinstance(address, list): assert (len(address) == ncomm) for i in range(ncomm): comm_list[i]['address'] = address[i] for i in range(ncomm): ikw = copy.deepcopy(kwargs) ikw.update(child_kwargs) ikw.update(comm_list[i]) ikw.setdefault('use_async', is_async) iname = ikw.pop('name') self.comm_list.append(get_comm(iname, **ikw)) self.eof_recv.append(0) self.eof_send.append(0) self.comm_list_backlog[i] = [] if ncomm > 0: kwargs['address'] = [x.address for x in self.comm_list] kwargs.update(noprop_kwargs) super(ForkComm, self).__init__(name, is_async=is_async, **kwargs) assert (not self.single_use) assert (not self.is_server) assert (not (self.is_client and (self.pattern != 'cycle')))