def test_asynchronous():
manager = conftest.Manager(Echo)
cxn = connect(**manager.kwargs)
echo = cxn.link('Echo')
# send a request that is ~110 MB
args = ['a' * int(1e6), 'b' * int(5e6), 'c' * int(1e7)]
kwargs = {
'1e6': 'x' * int(1e6),
'5e6': 'y' * int(5e6),
'array': list(range(int(1e7)))
}
future1 = echo.echo(*args, asynchronous=True, **kwargs)
# a few small requests
future2 = echo.echo('a', asynchronous=True)
future3 = echo.echo(data=list(range(10)), asynchronous=True)
# and a medium request
future4 = echo.echo(-2, -1, 0, q='q' * int(1e6), asynchronous=True)
assert isinstance(future1, concurrent.futures.Future)
assert isinstance(future2, concurrent.futures.Future)
assert isinstance(future3, concurrent.futures.Future)
assert isinstance(future4, concurrent.futures.Future)
assert future1.result(30) == [args, kwargs]
assert future2.result(30) == [['a'], {}]
assert future3.result(30) == [[], {'data': list(range(10))}]
assert future4.result(30) == [[-2, -1, 0], {'q': 'q' * int(1e6)}]
manager.shutdown(connection=cxn)
def test_invalid_username():
manager = conftest.Manager()
kwargs = manager.kwargs.copy()
kwargs['username'] = '******'
with pytest.raises(ValueError, match=r'Unregistered user'):
connect(**kwargs)
manager.shutdown()
def test_client_linkedclient_handlers():
values1 = []
values2 = []
values3 = []
values4 = []
def handler1(counter): # don't need to specify kwargs since none are emitted
values1.append(counter)
def handler2(counter): # don't need to specify kwargs since none are emitted
values2.append(counter)
def handler3(*args, **kwargs):
values3.append(1)
def handler4(*args, **kwargs):
values4.append(1)
manager = conftest.Manager(Echo, Heartbeat, add_heartbeat_task=True)
cxn = connect(**manager.kwargs)
link_hb = cxn.link('Heartbeat')
lc_hb = LinkedClient('Heartbeat', **manager.kwargs)
# the Echo Service does not emit notifications so make sure that the Manager
# does not route any notifications from Heartbeat to the links with Echo
link_echo = cxn.link('Echo')
link_echo.notification_handler = handler3
lc_echo = LinkedClient('Echo', **manager.kwargs)
lc_echo.notification_handler = handler4
assert link_echo.echo('foo', x=-1) == [['foo'], {'x': -1}]
assert lc_echo.echo('bar', 0) == [['bar', 0], {}]
link_hb.set_heart_rate(10)
link_hb.reset()
# the link will start to receive notifications 5 seconds earlier
link_hb.notification_handler = handler1
time.sleep(5)
lc_hb.reset()
lc_hb.notification_handler = handler2
time.sleep(5)
assert len(values1) > 30
assert len(values1) > len(values2) * 1.5 # ideally len(values1) == len(values2) * 2
assert len(values3) == 0 # the Echo Service does not emit notifications
assert len(values4) == 0 # the Echo Service does not emit notifications
assert values1.count(3) == 2 # the value 3 should appear twice since reset() was called twice
assert values2.count(3) == 1
assert link_echo.echo(foo='bar') == [[], {'foo': 'bar'}]
assert lc_echo.echo() == [[], {}]
link_hb.unlink()
lc_hb.unlink()
link_echo.disconnect() # disconnect is an alias for unlink
lc_echo.disconnect()
manager.shutdown(connection=cxn)
def test_not_json_serializable():
manager = conftest.Manager(Echo)
cxn = connect(**manager.kwargs)
e = cxn.link('Echo')
with pytest.raises(TypeError, match=r'not JSON serializable'):
e.echo(1 + 4j)
manager.shutdown(connection=cxn)
def test_invalid_password():
manager = conftest.Manager()
kwargs = manager.kwargs.copy()
kwargs['password'] = '******'
with pytest.raises(ValueError, match=r'Wrong login password'):
connect(**kwargs)
manager.shutdown()
def test_tls_disabled():
manager = conftest.Manager(disable_tls=False)
kwargs = manager.kwargs.copy()
kwargs['disable_tls'] = True
with pytest.raises(ConnectionError, match=r'You have TLS disabled'):
connect(**kwargs, timeout=5)
manager.shutdown()
def test_invalid_manager_password():
manager = conftest.Manager(password_manager='asdvgbaw4bn')
kwargs = manager.kwargs.copy()
kwargs['password_manager'] = 'x'
with pytest.raises(ValueError, match=r'Wrong Manager password'):
connect(**kwargs)
manager.shutdown()
def test_admin_requests():
manager = conftest.Manager()
cxn = connect(**manager.kwargs)
assert cxn.admin_request('port') == manager.port
assert cxn.admin_request('password') is None
assert cxn.admin_request('login')
assert cxn.admin_request('hostnames') is None
assert cxn.admin_request('users_table.is_user_registered', manager.admin_username) is True
assert cxn.admin_request('users_table.is_password_valid', manager.admin_username, manager.admin_password) is True
assert cxn.admin_request('users_table.is_admin', manager.admin_username) is True
assert cxn.admin_request('users_table.is_user_registered', 'no one special') is False
conns = cxn.admin_request('connections_table.connections')
assert len(conns) == 2
assert conns[0][4] == cxn.port
assert conns[0][5] == 'new connection request'
assert conns[1][4] == cxn.port
assert conns[1][5] == 'connected as a client'
hostnames = cxn.admin_request('hostnames_table.hostnames')
for alias in LOCALHOST_ALIASES:
assert alias in hostnames
with pytest.raises(ValueError, match=r'Cannot make asynchronous requests'):
cxn.admin_request('users_table.usernames', asynchronous=True)
manager.shutdown(connection=cxn)
def test_tls_enabled():
manager = conftest.Manager(disable_tls=True)
kwargs = manager.kwargs.copy()
kwargs['disable_tls'] = False
with pytest.raises(ConnectionError,
match=r'Try setting disable_tls=True$'):
connect(**kwargs)
manager.shutdown()
def test_wrong_port():
manager = conftest.Manager()
kwargs = manager.kwargs.copy()
kwargs['port'] = manager.get_available_port()
match = r'Cannot connect to {}:{}$'.format(constants.HOSTNAME,
kwargs['port'])
with pytest.raises(ConnectionError, match=match):
connect(timeout=100, **kwargs)
manager.shutdown()
def test_hostname_mismatch(host):
a = cryptography.x509.NameAttribute
o = cryptography.x509.NameOID
name = cryptography.x509.Name([a(o.COMMON_NAME, 'MSLNZ12345')])
manager = conftest.Manager(cert_common_name=name)
kwargs = manager.kwargs.copy()
kwargs['host'] = host
with pytest.raises(ConnectionError, match=r'set assert_hostname=False'):
connect(assert_hostname=True, **kwargs)
cxn = connect(assert_hostname=False, **kwargs)
manager.shutdown(connection=cxn)
def test_no_certificate_tls_disabled():
manager = conftest.Manager(disable_tls=True)
os.remove(manager.cert_file)
assert not os.path.isfile(manager.cert_file)
kwargs = manager.kwargs.copy()
kwargs['disable_tls'] = False
kwargs['cert_file'] = None
with pytest.raises(ConnectionError,
match=r'Try setting disable_tls=True$'):
connect(**kwargs)
manager.shutdown()
def test_unlink_client_max1():
manager = conftest.Manager(Echo, max_clients=1)
cxn1 = connect(**manager.kwargs)
cxn2 = connect(**manager.kwargs)
link1 = cxn1.link('Echo')
assert repr(link1).startswith("<Link with Echo[")
assert link1.service_name == 'Echo'
assert link1.echo(1, x=2) == [[1], {'x': 2}]
# the same Client can re-link to the same Service
link1b = cxn1.link('Echo')
assert link1b is not link1
assert link1b.service_name == 'Echo'
assert link1b.echo(1, x=2) == [[1], {'x': 2}]
# another Client cannot link
with pytest.raises(
RuntimeError,
match=r'The maximum number of Clients are already linked'):
cxn2.link('Echo')
link1.unlink()
assert repr(link1).startswith("<Un-Linked from Echo[")
assert link1._client is None
with pytest.raises(
AttributeError,
match=r"Cannot access 'echo' since the link has been broken"):
link1.echo(1)
# another Client can now link
link2 = cxn2.link('Echo')
assert link2.service_name == 'Echo'
assert link2.echo(1, x=2) == [[1], {'x': 2}]
link2.unlink()
assert link2._client is None
with pytest.raises(
AttributeError,
match=r"Cannot access 'echo' since the link has been broken"):
link2.echo(1)
# un-linking multiple times is okay
for i in range(20):
link2.unlink()
link2.disconnect() # an alias for unlink
manager.shutdown(connection=cxn1)
# shutting down the manager using cxn1 will also disconnect cxn2
assert not cxn1.is_connected()
assert not cxn2.is_connected()
def test_no_certificate():
# calling connect() will automatically get the certificate from the server
manager = conftest.Manager(disable_tls=False)
cert_file = os.path.join(constants.CERT_DIR, constants.HOSTNAME + '.crt')
assert manager.cert_file == cert_file
os.remove(manager.cert_file)
assert not os.path.isfile(manager.cert_file)
assert not os.path.isfile(cert_file)
kwargs = manager.kwargs.copy()
kwargs['cert_file'] = None
kwargs['auto_save'] = True
cxn = connect(**kwargs)
assert os.path.isfile(cert_file)
os.remove(cert_file)
manager.shutdown(connection=cxn)
def test_wrong_certificate():
manager = conftest.Manager()
key = os.path.join(tempfile.gettempdir(), '.msl', 'wrong-certificate.key')
cert = os.path.join(tempfile.gettempdir(), '.msl', 'wrong-certificate.crt')
assert cryptography.generate_key(path=key) == key
assert cryptography.generate_certificate(path=cert, key_path=key) == cert
kwargs = manager.kwargs.copy()
kwargs['cert_file'] = cert
with pytest.raises(ConnectionError) as e:
connect(**kwargs)
msg = str(e.value)
assert 'Perhaps the Network Manager is using a new certificate' in msg
assert '{}:{}'.format(constants.HOSTNAME, kwargs['port']) in msg
assert 'wrong-certificate.crt' in msg
os.remove(key)
os.remove(cert)
manager.shutdown()
def test_from_client():
# this tests that the Manager can handle multiple
# requests from a Client in a single network packet
manager = conftest.Manager(Echo, disable_tls=True)
with socket.socket() as sock:
sock.settimeout(5)
sock.connect(('localhost', manager.port))
# send all data as though the Client is connected via a terminal
# receive the "username" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'username'
sock.sendall(manager.admin_username.encode() + TERMINATION)
# receive the "password" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'password'
sock.sendall(manager.admin_password.encode() + TERMINATION)
# receive the "identity" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'identity'
sock.sendall(b'client' + TERMINATION)
# link with Echo
sock.sendall(b'link Echo' + TERMINATION)
reply = json.loads(sock.recv(1024).decode())
assert 'echo' in reply['result']['attributes']
# send multiple lines to the Manager
sock.sendall(b'Echo echo 1' + TERMINATION + b'Echo echo 1 2' +
TERMINATION + b'Echo echo x=3' + TERMINATION)
replies = []
while len(replies) < 3:
received = sock.recv(1024).split(TERMINATION)
replies.extend([json.loads(r.decode()) for r in received if r])
assert replies[0]['result'] == [[1], {}]
assert replies[1]['result'] == [[1, 2], {}]
assert replies[2]['result'] == [[], {'x': 3}]
manager.shutdown()
def test_synchronous():
manager = conftest.Manager(Echo)
cxn = connect(**manager.kwargs)
echo = cxn.link('Echo')
# send a request that is ~110 MB
args = ['a' * int(1e6), 'b' * int(5e6), 'c' * int(1e7)]
kwargs = {
'1e6': 'x' * int(1e6),
'5e6': 'y' * int(5e6),
'array': list(range(int(1e7)))
}
reply = echo.echo(*args, **kwargs)
assert reply[0] == args
assert reply[1] == kwargs
manager.shutdown(connection=cxn)
def test_unlink_client_max10():
manager = conftest.Manager(Echo, max_clients=10)
clients = [connect(**manager.kwargs) for _ in range(10)]
links = [client.link('Echo') for client in clients]
for link in links:
assert link.service_name == 'Echo'
assert link.echo(1, x=2) == [[1], {'x': 2}]
cxn = connect(**manager.kwargs)
# another Client cannot link
with pytest.raises(
RuntimeError,
match=r'The maximum number of Clients are already linked'):
cxn.link('Echo')
links[0].unlink()
assert links[0]._client is None
with pytest.raises(
AttributeError,
match=r"Cannot access 'echo' since the link has been broken"):
links[0].echo(1)
# another Client can now link
link2 = cxn.link('Echo')
assert link2.service_name == 'Echo'
assert link2.echo(1, x=2) == [[1], {'x': 2}]
link2.unlink()
assert link2._client is None
with pytest.raises(
AttributeError,
match=r"Cannot access 'echo' since the link has been broken"):
link2.echo(1)
manager.shutdown(connection=cxn)
# shutting down the manager using cxn will also disconnect all clients
for client in clients:
assert not client.is_connected()
def test_linked_echo():
manager = conftest.Manager(Echo)
manager.kwargs['name'] = 'foobar'
link = LinkedClient('Echo', **manager.kwargs)
args, kwargs = link.echo(1, 2, 3)
assert len(args) == 3
assert args[0] == 1
assert args[1] == 2
assert args[2] == 3
assert len(kwargs) == 0
args, kwargs = link.echo(x=4, y=5, z=6)
assert len(args) == 0
assert kwargs['x'] == 4
assert kwargs['y'] == 5
assert kwargs['z'] == 6
args, kwargs = link.echo(1, 2, 3, x=4, y=5, z=6)
assert len(args) == 3
assert args[0] == 1
assert args[1] == 2
assert args[2] == 3
assert kwargs['x'] == 4
assert kwargs['y'] == 5
assert kwargs['z'] == 6
assert len(link.service_attributes) == 2
assert 'echo' in link.service_attributes
assert 'set_logging_level' in link.service_attributes
assert link.name == 'foobar'
with pytest.raises(RuntimeError):
link.does_not_exist()
manager.shutdown(connection=link.client)
def test_from_service():
# this tests that the Manager can handle multiple
# replies from a Service in a single network packet
service_connected = []
notifications = []
def create_socket_service():
name = 'ManualService'
with socket.socket() as sock:
sock.settimeout(5)
sock.connect(('localhost', manager.port))
service_connected.append(True)
# receive the "username" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'username'
sock.sendall(manager.admin_username.encode() + TERMINATION)
# receive the "password" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'password'
sock.sendall(manager.admin_password.encode() + TERMINATION)
# receive the "identity" request
request = json.loads(sock.recv(1024).decode())
assert request['attribute'] == 'identity'
sock.sendall(
json.dumps({
'error': False,
'result': {
'type': 'service',
'name': name,
'attributes': {
'multiple': ''
},
},
'requester': request['requester'],
'uid': request['uid'],
}).encode() + TERMINATION)
# receive the request from the Client
request = json.loads(sock.recv(1024).decode())
response = json.dumps({
'error': False,
'result': 'the request was a success!',
'requester': request['requester'],
'uid': request['uid']
}).encode()
notify1 = json.dumps({
'error': False,
'result': [[1], {
'a': 1
}],
'service': name,
'uid': NOTIFICATION_UID,
}).encode()
notify2 = json.dumps({
'error': False,
'result': [[2], {
'b': 2
}],
'service': name,
'uid': NOTIFICATION_UID,
}).encode()
sock.sendall(notify1 + TERMINATION + notify2 + TERMINATION +
response + TERMINATION)
# wait for the Manager to shutdown
sock.recv(1024)
def handle_notification(*args, **kwargs):
notifications.append([args, kwargs])
# start the Manager
manager = conftest.Manager(disable_tls=True)
# start the Service
t = threading.Thread(target=create_socket_service, daemon=True)
t.start()
while not service_connected:
time.sleep(0.1)
# perform the test
cxn = connect(**manager.kwargs)
link = cxn.link('ManualService')
link.notification_handler = handle_notification
reply = link.multiple()
assert reply == 'the request was a success!'
assert notifications == [[(1, ), {'a': 1}], [(2, ), {'b': 2}]]
manager.shutdown(connection=cxn)
def test_manager_identity():
manager = conftest.Manager(BasicMath, MyArray, Echo)
cxn = connect(name='A.B.C', **manager.kwargs)
os = '{} {} {}'.format(platform.system(), platform.release(), platform.machine())
language = 'Python ' + platform.python_version()
identities = cxn.identities()
assert identities['hostname'] == HOSTNAME
assert identities['port'] == manager.port
assert identities['attributes'] == {
'identity': '() -> dict',
'link': '(service: str) -> bool'
}
assert identities['language'] == language
assert identities['os'] == os
assert 'A.B.C[{}:{}]'.format(HOSTNAME, cxn.port) in identities['clients']
assert 'BasicMath' in identities['services']
assert 'Echo' in identities['services']
assert 'MyArray' in identities['services']
identities = cxn.identities(as_string=True)
expected = r'''Manager\[{hostname}:\d+]
attributes:
identity\(\) -> dict
link\(service: str\) -> bool
language: {language}
os: {os}
Clients \[1]:
A.B.C\[{hostname}:\d+]
language: {language}
os: {os}
Services \[3]:
BasicMath\[{hostname}:\d+]
attributes:
add\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float]
divide\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float]
ensure_positive\(x:\s?Union\[int, float]\) -> bool
euler\(\) -> 2.718281828459045
multiply\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float]
pi\(\) -> 3.141592653589793
power\(x:\s?Union\[int, float], n=2\) -> Union\[int, float]
set_logging_level\(level:\s?Union\[str, int]\) -> bool
subtract\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float]
language: {language}
max_clients: -1
os: {os}
Echo\[{hostname}:\d+]
attributes:
echo\(\*args, \*\*kwargs\)
set_logging_level\(level:\s?Union\[str, int]\) -> bool
language: {language}
max_clients: -1
os: {os}
MyArray\[{hostname}:\d+]
attributes:
linspace\(start:\s?Union\[int, float], stop:\s?Union\[int, float], n=100\) -> List\[float]
scalar_multiply\(scalar:\s?Union\[int, float], data:\s?List\[float]\) -> List\[float]
set_logging_level\(level:\s?Union\[str, int]\) -> bool
language: {language}
max_clients: -1
os: {os}
'''.format(hostname=HOSTNAME, language=language, os=os).splitlines()
id_lines = identities.splitlines()
assert len(id_lines) == len(expected)
for pattern, string in zip(expected, id_lines):
assert re.match(pattern, string)
manager.shutdown(connection=cxn)