def test_double_dynamic_routing_zmqlet():
args1 = get_args()
args2 = get_args()
args3 = get_args()
logger = JinaLogger('zmq-test')
with Zmqlet(args1, logger) as z1, Zmqlet(args2, logger) as z2, Zmqlet(
args3, logger
) as z3:
assert z1.msg_sent == 0
assert z2.msg_sent == 0
assert z3.msg_sent == 0
req = jina_pb2.RequestProto()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
routing_table = {
'active_pod': 'executor1',
'pods': {
'executor1': {
'host': __default_host__,
'port': args1.port_in,
'expected_parts': 0,
'out_edges': [{'pod': 'executor2'}, {'pod': 'executor3'}],
},
'executor2': {
'host': __default_host__,
'port': args2.port_in,
'expected_parts': 1,
'out_edges': [],
},
'executor3': {
'host': __default_host__,
'port': args3.port_in,
'expected_parts': 1,
'out_edges': [],
},
},
}
msg.envelope.routing_table.CopyFrom(RoutingTable(routing_table).proto)
number_messages = 100
trips = 10
for i in range(trips):
for j in range(number_messages):
z1.send_message(msg)
time.sleep(1)
for i in range(number_messages):
z2.recv_message(callback)
z3.recv_message(callback)
total_number_messages = number_messages * trips
assert z1.msg_sent == 2 * total_number_messages
assert z2.msg_sent == 0
assert z2.msg_recv == total_number_messages
assert z3.msg_sent == 0
assert z3.msg_recv == total_number_messages
def test_remote_local_dynamic_routing_zmqlet():
args1 = get_args()
args2 = get_args()
args2.zmq_identity = 'test-identity'
args2.hosts_in_connect = [f'{args1.host}:{args1.port_out}']
logger = JinaLogger('zmq-test')
with Zmqlet(args1, logger) as z1, Zmqlet(args2, logger) as z2:
assert z1.msg_sent == 0
assert z1.msg_recv == 0
assert z2.msg_sent == 0
assert z2.msg_recv == 0
req = jina_pb2.RequestProto()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
routing_pb = jina_pb2.RoutingTableProto()
routing_table = {
'active_pod': 'pod1',
'pods': {
'pod1': {
'host': '0.0.0.0',
'port': args1.port_in,
'expected_parts': 0,
'out_edges': [{
'pod': 'pod2',
'send_as_bind': True
}],
},
'pod2': {
'host': '0.0.0.0',
'port': args2.port_in,
'expected_parts': 1,
'out_edges': [],
'target_identity': args2.zmq_identity,
},
},
}
json_format.ParseDict(routing_table, routing_pb)
msg.envelope.routing_table.CopyFrom(routing_pb)
z2.recv_message(callback)
assert z2.msg_sent == 0
assert z2.msg_recv == 0
z1.send_message(msg)
z2.recv_message(callback)
assert z1.msg_sent == 1
assert z1.msg_recv == 0
assert z2.msg_sent == 0
assert z2.msg_recv == 1
def test_simple_zmqlet():
args = set_pea_parser().parse_args([
'--host-in', '0.0.0.0',
'--host-out', '0.0.0.0',
'--port-in', '12346',
'--port-out', '12347',
'--socket-in', 'PULL_CONNECT',
'--socket-out', 'PUSH_CONNECT',
'--timeout-ctrl', '-1'])
args2 = set_pea_parser().parse_args([
'--host-in', '0.0.0.0',
'--host-out', '0.0.0.0',
'--port-in', '12347',
'--port-out', '12346',
'--socket-in', 'PULL_BIND',
'--socket-out', 'PUSH_BIND',
'--uses', '_logforward',
'--timeout-ctrl', '-1'
])
logger = logging.getLogger('zmq-test')
with BasePea(args2) as z1, Zmqlet(args, logger) as z:
req = jina_pb2.RequestProto()
req.request_id = get_random_identity()
d = req.index.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
z.send_message(msg)
def __init__(self, args: Union['argparse.Namespace', Dict]):
super().__init__()
self.args = args
self.name = self.__class__.__name__ #: this is the process name
self.is_ready_event = _get_event(self)
self.is_shutdown = _get_event(self)
self.ready_or_shutdown = _make_or_event(self, self.is_ready_event,
self.is_shutdown)
self.is_shutdown.clear()
if 'daemon' in args:
self.daemon = args.daemon
if 'name' in self.args and self.args.name:
self.name = f'runtime-{self.args.name}'
if 'role' in self.args and self.args.role == PeaRoleType.PARALLEL:
self.name = f'runtime-{self.args.name}-{self.args.pea_id}'
if 'role' in self.args and self.args.role == PeaRoleType.HEAD:
self.name = f'runtime-{self.args.name}-head'
if 'role' in self.args and self.args.role == PeaRoleType.TAIL:
self.name = f'runtime-{self.args.name}-tail'
if 'host' in self.args and 'port_ctrl' in self.args and 'ctrl_with_ipc' in self.args:
self.ctrl_addr, self.ctrl_with_ipc = Zmqlet.get_ctrl_address(
self.args.host, self.args.port_ctrl, self.args.ctrl_with_ipc)
if 'log_id' in self.args and 'log_config' in self.args:
self.logger = JinaLogger(self.name,
log_id=self.args.log_id,
log_config=self.args.log_config)
else:
self.logger = JinaLogger(self.name)
def test_simple_zmqlet(self):
args = set_pea_parser().parse_args([
'--host-in', '0.0.0.0',
'--host-out', '0.0.0.0',
'--port-in', '12346',
'--port-out', '12347',
'--socket-in', 'PULL_CONNECT',
'--socket-out', 'PUSH_CONNECT',
'--timeout-ctrl', '-1'])
args2 = set_pea_parser().parse_args([
'--host-in', '0.0.0.0',
'--host-out', '0.0.0.0',
'--port-in', '12347',
'--port-out', '12346',
'--socket-in', 'PULL_BIND',
'--socket-out', 'PUSH_BIND',
'--uses', '_logforward',
'--timeout-ctrl', '-1'
])
with BasePea(args2) as z1, Zmqlet(args) as z:
req = jina_pb2.Request()
req.request_id = 1
d = req.index.docs.add()
d.id = 2
msg = add_envelope(req, 'tmp', '')
z.send_message(msg)
def test_simple_dynamic_routing_zmqlet():
args1 = get_args()
args2 = get_args()
logger = JinaLogger('zmq-test')
with Zmqlet(args1, logger) as z1, Zmqlet(args2, logger) as z2:
assert z1.msg_sent == 0
assert z1.msg_recv == 0
assert z2.msg_sent == 0
assert z2.msg_recv == 0
req = jina_pb2.RequestProto()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
routing_pb = jina_pb2.RoutingTableProto()
routing_table = {
'active_pod': 'executor1',
'pods': {
'executor1': {
'host': __default_host__,
'port': args1.port_in,
'expected_parts': 0,
'out_edges': [{'pod': 'executor2'}],
},
'executor2': {
'host': __default_host__,
'port': args2.port_in,
'expected_parts': 1,
'out_edges': [],
},
},
}
json_format.ParseDict(routing_table, routing_pb)
msg.envelope.routing_table.CopyFrom(routing_pb)
z2.recv_message(callback)
assert z2.msg_sent == 0
assert z2.msg_recv == 0
z1.send_message(msg)
z2.recv_message(callback)
assert z1.msg_sent == 1
assert z1.msg_recv == 0
assert z2.msg_sent == 0
assert z2.msg_recv == 1
def test_not_read_zmqlet():
with MockBasePeaNotRead(args3), Zmqlet(args1, default_logger) as z:
req = jina_pb2.RequestProto()
req.request_id = get_random_identity()
d = req.index.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
z.send_message(msg)
def test_read_zmqlet():
with MockBasePeaRead(args2), Zmqlet(args1, default_logger) as z:
req = jina_pb2.RequestProto()
req.request_id = uuid.uuid1().hex
d = req.index.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
z.send_message(msg)
def test_read_zmqlet():
with MockBasePeaRead(args2), Zmqlet(args1, default_logger) as z:
req = Request()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
z.send_message(msg)
def test_recv_message_zmqlet(mocker):
zmqlet1 = Zmqlet(args1, default_logger)
zmqlet2 = Zmqlet(args2, default_logger)
req = Request()
req.request_id = random_identity()
doc = req.data.docs.add()
doc.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
def callback(msg_):
assert msg_.request.docs[0].tags['id'] == msg.request.data.docs[0].tags['id']
mock = mocker.Mock()
zmqlet1.send_message(msg)
time.sleep(1)
zmqlet2.recv_message(mock)
validate_callback(mock, callback)
def test_not_decompressed_zmqlet(mocker):
with MockPea(args2) as pea, Zmqlet(args1, default_logger) as z:
req = Request()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
mock = mocker.Mock()
z.send_message(msg)
time.sleep(1)
z.recv_message(mock)
def callback(msg_):
pass
validate_callback(mock, callback)
print(f' joining pea')
pea.join()
print(f' joined pea')
def test_simple_zmqlet():
args = set_pea_parser().parse_args(
[
'--host-in',
'0.0.0.0',
'--host-out',
'0.0.0.0',
'--socket-in',
'PULL_CONNECT',
'--socket-out',
'PUSH_CONNECT',
'--timeout-ctrl',
'-1',
]
)
args2 = set_pea_parser().parse_args(
[
'--host-in',
'0.0.0.0',
'--host-out',
'0.0.0.0',
'--port-in',
str(args.port_out),
'--port-out',
str(args.port_in),
'--socket-in',
'PULL_BIND',
'--socket-out',
'PUSH_BIND',
'--timeout-ctrl',
'-1',
]
)
logger = JinaLogger('zmq-test')
with BasePea(args2), Zmqlet(args, logger) as z:
req = jina_pb2.RequestProto()
req.request_id = random_identity()
d = req.data.docs.add()
d.tags['id'] = 2
msg = Message(None, req, 'tmp', '')
z.send_message(msg)
def test_get_ctrl_addr(host):
p = set_pea_parser().parse_args(['--host', '[email protected]', '--port-ctrl', '56789'])
assert Zmqlet.get_ctrl_address(p)[0] == 'tcp://192.0.0.1:56789'
def test_get_ctrl_addr_ipc(host):
assert Zmqlet.get_ctrl_address(host, 56789, True)[0].startswith('ipc')
def test_get_ctrl_addr(host):
assert Zmqlet.get_ctrl_address(host, 56789,
False)[0] == 'tcp://192.0.0.1:56789'
def test_get_ctrl_addr_ipc(host):
assert 'ipc' == Zmqlet.get_ctrl_address(host, 56789, True)[0][0:3]