def setup(self):
r"""Start driver and instance."""
if self.direction is None: # pragma: debug
return
nprev_comm = self.comm_count
nprev_thread = self.thread_count
nprev_fd = self.fd_count
# File
if self.is_file and (self.direction == 'input'):
with open(self.filename, 'wb') as fd:
fd.write(self.testing_options['contents'])
assert(os.path.isfile(self.filename))
# Drivers
args, kwargs = self.iodriver_args
self.iodriver = self.iodriver_class(*args, **kwargs)
self.iodriver.start()
self.iodriver.wait_for_loop()
# Test comm
if not self.is_file:
if self.direction == 'input':
kws = self.iodriver.icomm.opp_comm_kwargs()
kws.update(self.test_comm_kwargs)
self.test_comm = get_comm('in', **kws)
elif self.direction == 'output':
kws = self.iodriver.ocomm.opp_comm_kwargs()
kws.update(self.test_comm_kwargs)
self.test_comm = get_comm('out', **kws)
# Test class
super(TestBase, self).setup(nprev_comm=nprev_comm,
nprev_thread=nprev_thread,
nprev_fd=nprev_fd)
def __init__(self, name, comm=None, **kwargs):
self.comm_list = []
self.curr_comm_index = 0
self.eof_recv = []
address = kwargs.pop('address', None)
if (comm in [None, 'ForkComm']):
if isinstance(address, list):
ncomm = len(address)
else:
ncomm = 0
comm = [None for i in range(ncomm)]
assert (isinstance(comm, list))
ncomm = len(comm)
for i in range(ncomm):
if comm[i] is None:
comm[i] = {}
if comm[i].get('name', None) is None:
comm[i]['name'] = get_comm_name(name, i)
if isinstance(address, list):
assert (len(address) == ncomm)
for i in range(ncomm):
comm[i]['address'] = address[i]
for i in range(ncomm):
ikw = dict(**kwargs)
ikw.update(**comm[i])
iname = ikw.pop('name')
self.comm_list.append(get_comm(iname, **ikw))
self.eof_recv.append(0)
if ncomm > 0:
kwargs['address'] = [x.address for x in self.comm_list]
kwargs['comm'] = 'ForkComm'
super(ForkComm, self).__init__(name, **kwargs)
assert (not self.single_use)
assert (not self.is_server)
assert (not self.is_client)
def __init__(self,
name,
request_comm=None,
response_kwargs=None,
dont_open=False,
**kwargs):
if response_kwargs is None:
response_kwargs = dict()
icomm_name = name
icomm_kwargs = kwargs
icomm_kwargs['direction'] = 'recv'
icomm_kwargs['dont_open'] = True
icomm_kwargs['comm'] = request_comm
self.response_kwargs = response_kwargs
self.icomm = get_comm(icomm_name, **icomm_kwargs)
self.ocomm = OrderedDict()
self.response_kwargs.setdefault('comm', self.icomm.comm_class)
self.response_kwargs.setdefault('recv_timeout',
self.icomm.recv_timeout)
self.response_kwargs.setdefault('language', self.icomm.language)
self._used_response_comms = dict()
self.clients = []
self.closed_clients = []
self.nclients_expected = int(os.environ.get('YGG_NCLIENTS', 0))
super(ServerComm, self).__init__(self.icomm.name,
dont_open=dont_open,
recv_timeout=self.icomm.recv_timeout,
is_interface=self.icomm.is_interface,
direction='recv',
no_suffix=True,
address=self.icomm.address)
def __init__(self,
name,
request_comm=None,
response_kwargs=None,
dont_open=False,
**kwargs):
if response_kwargs is None:
response_kwargs = dict()
ocomm_name = name
ocomm_kwargs = kwargs
ocomm_kwargs['direction'] = 'send'
ocomm_kwargs['dont_open'] = True
ocomm_kwargs['comm'] = request_comm
self.response_kwargs = response_kwargs
self.ocomm = get_comm(ocomm_name, **ocomm_kwargs)
self.icomm = dict()
self.icomm_order = []
self.response_kwargs.setdefault('comm', self.ocomm.comm_class)
self.response_kwargs.setdefault('recv_timeout',
self.ocomm.recv_timeout)
self.response_kwargs.setdefault('language', self.ocomm.language)
super(ClientComm, self).__init__(self.ocomm.name,
dont_open=dont_open,
recv_timeout=self.ocomm.recv_timeout,
is_interface=self.ocomm.is_interface,
direction='send',
no_suffix=True,
address=self.ocomm.address)
def create_response_comm(self, header):
r"""Create a response comm based on information from the last header."""
if not isinstance(header, dict): # pragma: debug
raise RuntimeError("No header received with last message.")
elif 'response_address' not in header: # pragma: debug
raise RuntimeError(
"Last header does not contain response address.")
comm_kwargs = dict(address=header['response_address'],
direction='send',
**self.response_kwargs)
if self.direct_connection:
comm_kwargs['is_response_client'] = True
else:
comm_kwargs['is_response_server'] = True
response_id = header['request_id']
while response_id in self.requests: # pragma: debug
response_id += str(uuid.uuid4())
header['response_id'] = response_id
if header['response_address'] not in self.ocomm:
self.ocomm[header['response_address']] = get_comm(
self.name + '.server_response_comm.' + response_id,
**comm_kwargs)
client_model = header.get('model', '')
self.ocomm[header['response_address']].client_model = client_model
if client_model and (client_model not in self.clients):
self.clients.append(client_model)
self.requests[response_id] = Request(header['response_address'],
header['request_id'])
def rank_comm(self, name, mpi_rank, rank_kwargs, close_comm):
r"""MPI communicator for rank > 0 processes."""
if mpi_rank > 0:
x = get_comm(name, **rank_kwargs)
yield x
close_comm(x)
else:
yield None
def test_comm(self, iodriver, direction, filecomm, test_comm_kwargs,
close_comm):
r"""Communicator for testing."""
test_comm = None
if not filecomm:
if direction == 'input':
kws = iodriver.icomm.opp_comm_kwargs()
kws.update(test_comm_kwargs)
test_comm = get_comm('in', **kws)
elif direction == 'output':
kws = iodriver.ocomm.opp_comm_kwargs()
kws.update(test_comm_kwargs)
test_comm = get_comm('out', **kws)
try:
yield test_comm
finally:
if test_comm is not None:
close_comm(test_comm)
def create_recv_comm(self, name, commtype, send_comm, testing_options,
**kwargs):
r"""Create a receive communicator."""
kws = self.get_recv_comm_kwargs(commtype, send_comm, testing_options,
**kwargs)
x = get_comm(name, **kws)
assert (x.is_open)
x.drain_server_signon_messages()
return x
def setup(self):
r"""Start driver and instance."""
if self.direction is None: # pragma: debug
return
nprev_comm = self.comm_count
nprev_thread = self.thread_count
nprev_fd = self.fd_count
idriver_class = self.idriver_class
odriver_class = self.odriver_class
# File
if self.is_file and (self.direction == 'input'):
with open(self.filename, 'wb') as fd:
fd.write(self.testing_options['contents'])
# Drivers
comm_env = None
if idriver_class is not None:
args, kwargs = self.idriver_args
self.idriver = idriver_class(*args, **kwargs)
self.idriver.start()
comm_env = self.idriver.comm_env
if odriver_class is not None:
args, kwargs = self.odriver_args
if comm_env is not None:
kwargs['comm_env'] = comm_env
self.odriver = odriver_class(*args, **kwargs)
self.odriver.start()
# Test comm
if self.direction == 'input':
os.environ.update(self.idriver.env)
if self.odriver is not None:
kws = self.odriver.icomm.opp_comm_kwargs()
kws.update(self.test_comm_kwargs)
self.test_comm = get_comm('in', **kws)
elif self.direction == 'output':
os.environ.update(self.odriver.env)
if self.idriver is not None:
kws = self.idriver.ocomm.opp_comm_kwargs()
kws.update(self.test_comm_kwargs)
self.test_comm = get_comm('out', **kws)
# Test class
super(TestBase, self).setup(nprev_comm=nprev_comm,
nprev_thread=nprev_thread,
nprev_fd=nprev_fd)
def create_response_comm(self):
r"""Create a response comm based on information from the last header."""
if not isinstance(self.icomm._last_header, dict): # pragma: debug
raise RuntimeError("No header received with last message.")
elif 'response_address' not in self.icomm._last_header: # pragma: debug
raise RuntimeError("Last header does not contain response address.")
comm_kwargs = dict(address=self.icomm._last_header['response_address'],
direction='send', is_response_server=True,
single_use=True, **self.response_kwargs)
self.ocomm = get_comm(self.name + '.server_response_comm',
**comm_kwargs)
def __init__(self, name, comm_list=None, is_async=False, **kwargs):
child_keys = ['serializer_class', 'serializer_kwargs', # 'datatype',
'format_str', 'field_names', 'field_units', 'as_array']
noprop_keys = ['send_converter', 'recv_converter', 'filter', 'transform']
child_kwargs = {k: kwargs.pop(k) for k in child_keys if k in kwargs}
noprop_kwargs = {k: kwargs.pop(k) for k in noprop_keys if k in kwargs}
self.comm_list = []
self.curr_comm_index = 0
self.eof_recv = []
address = kwargs.pop('address', None)
if comm_list is None:
if isinstance(address, list):
ncomm = len(address)
else:
ncomm = 0
comm_list = [None for i in range(ncomm)]
assert(isinstance(comm_list, list))
ncomm = len(comm_list)
for i in range(ncomm):
if comm_list[i] is None:
comm_list[i] = {}
if comm_list[i].get('name', None) is None:
comm_list[i]['name'] = get_comm_name(name, i)
for k in child_kwargs.keys():
if k in comm_list[i]: # pragma: debug
raise ValueError("The keyword '%s' was specified for both the "
"root ForkComm and a child comm, but can only "
"be present for one." % k)
if isinstance(address, list):
assert(len(address) == ncomm)
for i in range(ncomm):
comm_list[i]['address'] = address[i]
for i in range(ncomm):
ikw = copy.deepcopy(kwargs)
ikw.update(child_kwargs)
ikw.update(comm_list[i])
ikw.setdefault('use_async', is_async)
iname = ikw.pop('name')
self.comm_list.append(get_comm(iname, **ikw))
self.eof_recv.append(0)
if ncomm > 0:
kwargs['address'] = [x.address for x in self.comm_list]
kwargs.update(noprop_kwargs)
super(ForkComm, self).__init__(name, is_async=is_async, **kwargs)
assert(not self.single_use)
assert(not self.is_server)
assert(not self.is_client)
def __init__(self,
name,
request_commtype=None,
response_kwargs=None,
dont_open=False,
is_async=False,
direct_connection=False,
**kwargs):
if response_kwargs is None:
response_kwargs = dict()
icomm_name = name
icomm_kwargs = kwargs
icomm_kwargs.update(direction='recv',
dont_open=True,
commtype=request_commtype)
icomm_kwargs.setdefault('is_server', True)
icomm_kwargs.setdefault('use_async', is_async)
if icomm_kwargs.get('use_async', False):
icomm_kwargs.setdefault('async_recv_method', 'recv_message')
self.direct_connection = direct_connection
self.response_kwargs = response_kwargs
self.icomm = get_comm(icomm_name, **icomm_kwargs)
self.ocomm = OrderedDict()
self.requests = OrderedDict()
self.response_kwargs.setdefault('is_interface',
self.icomm.is_interface)
self.response_kwargs.setdefault('commtype', self.icomm._commtype)
self.response_kwargs.setdefault('recv_timeout',
self.icomm.recv_timeout)
self.response_kwargs.setdefault('language', self.icomm.language)
self.response_kwargs.setdefault('use_async', self.icomm.is_async)
self.response_kwargs.setdefault('env', self.icomm.env)
self.clients = []
self.closed_clients = []
self.nclients_expected = int(os.environ.get('YGG_NCLIENTS', 0))
super(ServerComm, self).__init__(self.icomm.name,
dont_open=dont_open,
recv_timeout=self.icomm.recv_timeout,
is_interface=self.icomm.is_interface,
direction='recv',
no_suffix=True,
address=self.icomm.address,
is_async=self.icomm.is_async,
env=self.icomm.env)
def __init__(self,
name,
request_commtype=None,
response_kwargs=None,
dont_open=False,
is_async=False,
direct_connection=False,
**kwargs):
if response_kwargs is None:
response_kwargs = dict()
ocomm_name = name
ocomm_kwargs = kwargs
ocomm_kwargs['direction'] = 'send'
ocomm_kwargs['dont_open'] = True
ocomm_kwargs['commtype'] = request_commtype
ocomm_kwargs.setdefault('use_async', is_async)
if direct_connection:
ocomm_kwargs.setdefault('is_client', True)
self.direct_connection = direct_connection
self.response_kwargs = response_kwargs
self.ocomm = get_comm(ocomm_name, **ocomm_kwargs)
self.icomm = None
self.request_order = []
self.responses = OrderedDict()
for k, v in self.ocomm.get_response_comm_kwargs.items():
self.response_kwargs.setdefault(k, v)
self.response_kwargs.setdefault('is_interface',
self.ocomm.is_interface)
self.response_kwargs.setdefault('recv_timeout',
self.ocomm.recv_timeout)
self.response_kwargs.setdefault('language', self.ocomm.language)
self.response_kwargs.setdefault('use_async', self.ocomm.is_async)
self.response_kwargs.setdefault('env', self.ocomm.env)
super(ClientComm, self).__init__(self.ocomm.name,
dont_open=dont_open,
recv_timeout=self.ocomm.recv_timeout,
is_interface=self.ocomm.is_interface,
direction='send',
no_suffix=True,
address=self.ocomm.address,
is_async=self.ocomm.is_async,
env=self.ocomm.env)
def __init__(self, name, request_comm=None, response_kwargs=None,
dont_open=False, **kwargs):
if response_kwargs is None:
response_kwargs = dict()
icomm_name = name
icomm_kwargs = kwargs
icomm_kwargs['direction'] = 'recv'
icomm_kwargs['dont_open'] = True
icomm_kwargs['comm'] = request_comm
self.response_kwargs = response_kwargs
self.icomm = get_comm(icomm_name, **icomm_kwargs)
self.ocomm = None
self.response_kwargs.setdefault('comm', self.icomm.comm_class)
self.response_kwargs.setdefault('recv_timeout', self.icomm.recv_timeout)
self._used_response_comms = dict()
super(ServerComm, self).__init__(self.icomm.name, dont_open=dont_open,
recv_timeout=self.icomm.recv_timeout,
is_interface=self.icomm.is_interface,
direction='recv', no_suffix=True,
address=self.icomm.address)
def create_instance(self):
r"""Create a new instance of the class."""
inst = get_comm(*self.inst_args, **self.inst_kwargs)
assert (isinstance(inst, self.import_cls))
return inst
def __init__(self,
name,
comm_list=None,
is_async=False,
pattern=None,
**kwargs):
child_kwargs = {
k: kwargs.pop(k)
for k in self.child_keys if k in kwargs
}
noprop_kwargs = {
k: kwargs.pop(k)
for k in self.noprop_keys if k in kwargs
}
self.comm_list_backlog = {}
self.comm_list = []
self.curr_comm_index = 0
self.eof_recv = []
self.eof_send = []
if pattern is None:
if kwargs.get('direction', 'send') == 'recv':
pattern = 'cycle'
else:
pattern = 'broadcast'
self.pattern = pattern
if kwargs.get('direction', 'send') == 'recv':
assert (self.pattern in ['cycle', 'gather'])
else:
assert (self.pattern in ['cycle', 'scatter', 'broadcast'])
address = kwargs.pop('address', None)
if comm_list is None:
if isinstance(address, list):
ncomm = len(address)
else:
ncomm = 0
comm_list = [None for i in range(ncomm)]
assert (isinstance(comm_list, list))
ncomm = len(comm_list)
for i in range(ncomm):
if comm_list[i] is None:
comm_list[i] = {}
if comm_list[i].get('name', None) is None:
comm_list[i]['name'] = get_comm_name(name, i)
for k in child_kwargs.keys():
if k in comm_list[i]: # pragma: debug
raise ValueError(
"The keyword '%s' was specified for both the "
"root ForkComm and a child comm, but can only "
"be present for one." % k)
if isinstance(address, list):
assert (len(address) == ncomm)
for i in range(ncomm):
comm_list[i]['address'] = address[i]
for i in range(ncomm):
ikw = copy.deepcopy(kwargs)
ikw.update(child_kwargs)
ikw.update(comm_list[i])
ikw.setdefault('use_async', is_async)
iname = ikw.pop('name')
self.comm_list.append(get_comm(iname, **ikw))
self.eof_recv.append(0)
self.eof_send.append(0)
self.comm_list_backlog[i] = []
if ncomm > 0:
kwargs['address'] = [x.address for x in self.comm_list]
kwargs.update(noprop_kwargs)
super(ForkComm, self).__init__(name, is_async=is_async, **kwargs)
assert (not self.single_use)
assert (not self.is_server)
assert (not (self.is_client and (self.pattern != 'cycle')))
