def test_tls_disabled(): manager = conftest.Manager(disable_tls=False) kwargs = manager.kwargs.copy() kwargs['disable_tls'] = True with pytest.raises(ConnectionError, match=r'You have TLS disabled'): connect(**kwargs, timeout=5) manager.shutdown()
def test_no_manager_no_certificate_remotehost(): conftest.Manager.remove_files() host = 'MSLNZ12345' match = r'Cannot connect to {}:{} to get the certificate$'.format( host, constants.PORT) with pytest.raises(ConnectionError, match=match): connect(host=host, disable_tls=False)
def test_invalid_password(): manager = conftest.Manager() kwargs = manager.kwargs.copy() kwargs['password'] = '******' with pytest.raises(ValueError, match=r'Wrong login password'): connect(**kwargs) manager.shutdown()
def test_invalid_manager_password(): manager = conftest.Manager(password_manager='asdvgbaw4bn') kwargs = manager.kwargs.copy() kwargs['password_manager'] = 'x' with pytest.raises(ValueError, match=r'Wrong Manager password'): connect(**kwargs) manager.shutdown()
def test_invalid_username(): manager = conftest.Manager() kwargs = manager.kwargs.copy() kwargs['username'] = '******' with pytest.raises(ValueError, match=r'Unregistered user'): connect(**kwargs) manager.shutdown()
def test_tls_enabled(): manager = conftest.Manager(disable_tls=True) kwargs = manager.kwargs.copy() kwargs['disable_tls'] = False with pytest.raises(ConnectionError, match=r'Try setting disable_tls=True$'): connect(**kwargs) manager.shutdown()
def test_no_manager_timeout_asyncio(): timeout = 0.5 port = conftest.Manager.get_available_port() match = r'Cannot connect to {}:{} within {} seconds$'.format( constants.HOSTNAME, port, timeout) t0 = perf_counter() with pytest.raises(TimeoutError, match=match): connect(port=port, disable_tls=True, timeout=timeout) assert abs(timeout - (perf_counter() - t0)) < 0.2
def test_wrong_port(): manager = conftest.Manager() kwargs = manager.kwargs.copy() kwargs['port'] = manager.get_available_port() match = r'Cannot connect to {}:{}$'.format(constants.HOSTNAME, kwargs['port']) with pytest.raises(ConnectionError, match=match): connect(timeout=100, **kwargs) manager.shutdown()
def test_hostname_mismatch(host): a = cryptography.x509.NameAttribute o = cryptography.x509.NameOID name = cryptography.x509.Name([a(o.COMMON_NAME, 'MSLNZ12345')]) manager = conftest.Manager(cert_common_name=name) kwargs = manager.kwargs.copy() kwargs['host'] = host with pytest.raises(ConnectionError, match=r'set assert_hostname=False'): connect(assert_hostname=True, **kwargs) cxn = connect(assert_hostname=False, **kwargs) manager.shutdown(connection=cxn)
def test_unlink_client_max1(): manager = conftest.Manager(Echo, max_clients=1) cxn1 = connect(**manager.kwargs) cxn2 = connect(**manager.kwargs) link1 = cxn1.link('Echo') assert repr(link1).startswith("<Link with Echo[") assert link1.service_name == 'Echo' assert link1.echo(1, x=2) == [[1], {'x': 2}] # the same Client can re-link to the same Service link1b = cxn1.link('Echo') assert link1b is not link1 assert link1b.service_name == 'Echo' assert link1b.echo(1, x=2) == [[1], {'x': 2}] # another Client cannot link with pytest.raises( RuntimeError, match=r'The maximum number of Clients are already linked'): cxn2.link('Echo') link1.unlink() assert repr(link1).startswith("<Un-Linked from Echo[") assert link1._client is None with pytest.raises( AttributeError, match=r"Cannot access 'echo' since the link has been broken"): link1.echo(1) # another Client can now link link2 = cxn2.link('Echo') assert link2.service_name == 'Echo' assert link2.echo(1, x=2) == [[1], {'x': 2}] link2.unlink() assert link2._client is None with pytest.raises( AttributeError, match=r"Cannot access 'echo' since the link has been broken"): link2.echo(1) # un-linking multiple times is okay for i in range(20): link2.unlink() link2.disconnect() # an alias for unlink manager.shutdown(connection=cxn1) # shutting down the manager using cxn1 will also disconnect cxn2 assert not cxn1.is_connected() assert not cxn2.is_connected()
def test_no_certificate_tls_disabled(): manager = conftest.Manager(disable_tls=True) os.remove(manager.cert_file) assert not os.path.isfile(manager.cert_file) kwargs = manager.kwargs.copy() kwargs['disable_tls'] = False kwargs['cert_file'] = None with pytest.raises(ConnectionError, match=r'Try setting disable_tls=True$'): connect(**kwargs) manager.shutdown()
def test_asynchronous(): manager = conftest.Manager(Echo) cxn = connect(**manager.kwargs) echo = cxn.link('Echo') # send a request that is ~110 MB args = ['a' * int(1e6), 'b' * int(5e6), 'c' * int(1e7)] kwargs = { '1e6': 'x' * int(1e6), '5e6': 'y' * int(5e6), 'array': list(range(int(1e7))) } future1 = echo.echo(*args, asynchronous=True, **kwargs) # a few small requests future2 = echo.echo('a', asynchronous=True) future3 = echo.echo(data=list(range(10)), asynchronous=True) # and a medium request future4 = echo.echo(-2, -1, 0, q='q' * int(1e6), asynchronous=True) assert isinstance(future1, concurrent.futures.Future) assert isinstance(future2, concurrent.futures.Future) assert isinstance(future3, concurrent.futures.Future) assert isinstance(future4, concurrent.futures.Future) assert future1.result(30) == [args, kwargs] assert future2.result(30) == [['a'], {}] assert future3.result(30) == [[], {'data': list(range(10))}] assert future4.result(30) == [[-2, -1, 0], {'q': 'q' * int(1e6)}] manager.shutdown(connection=cxn)
def test_client_linkedclient_handlers(): values1 = [] values2 = [] values3 = [] values4 = [] def handler1(counter): # don't need to specify kwargs since none are emitted values1.append(counter) def handler2(counter): # don't need to specify kwargs since none are emitted values2.append(counter) def handler3(*args, **kwargs): values3.append(1) def handler4(*args, **kwargs): values4.append(1) manager = conftest.Manager(Echo, Heartbeat, add_heartbeat_task=True) cxn = connect(**manager.kwargs) link_hb = cxn.link('Heartbeat') lc_hb = LinkedClient('Heartbeat', **manager.kwargs) # the Echo Service does not emit notifications so make sure that the Manager # does not route any notifications from Heartbeat to the links with Echo link_echo = cxn.link('Echo') link_echo.notification_handler = handler3 lc_echo = LinkedClient('Echo', **manager.kwargs) lc_echo.notification_handler = handler4 assert link_echo.echo('foo', x=-1) == [['foo'], {'x': -1}] assert lc_echo.echo('bar', 0) == [['bar', 0], {}] link_hb.set_heart_rate(10) link_hb.reset() # the link will start to receive notifications 5 seconds earlier link_hb.notification_handler = handler1 time.sleep(5) lc_hb.reset() lc_hb.notification_handler = handler2 time.sleep(5) assert len(values1) > 30 assert len(values1) > len(values2) * 1.5 # ideally len(values1) == len(values2) * 2 assert len(values3) == 0 # the Echo Service does not emit notifications assert len(values4) == 0 # the Echo Service does not emit notifications assert values1.count(3) == 2 # the value 3 should appear twice since reset() was called twice assert values2.count(3) == 1 assert link_echo.echo(foo='bar') == [[], {'foo': 'bar'}] assert lc_echo.echo() == [[], {}] link_hb.unlink() lc_hb.unlink() link_echo.disconnect() # disconnect is an alias for unlink lc_echo.disconnect() manager.shutdown(connection=cxn)
def test_admin_requests(): manager = conftest.Manager() cxn = connect(**manager.kwargs) assert cxn.admin_request('port') == manager.port assert cxn.admin_request('password') is None assert cxn.admin_request('login') assert cxn.admin_request('hostnames') is None assert cxn.admin_request('users_table.is_user_registered', manager.admin_username) is True assert cxn.admin_request('users_table.is_password_valid', manager.admin_username, manager.admin_password) is True assert cxn.admin_request('users_table.is_admin', manager.admin_username) is True assert cxn.admin_request('users_table.is_user_registered', 'no one special') is False conns = cxn.admin_request('connections_table.connections') assert len(conns) == 2 assert conns[0][4] == cxn.port assert conns[0][5] == 'new connection request' assert conns[1][4] == cxn.port assert conns[1][5] == 'connected as a client' hostnames = cxn.admin_request('hostnames_table.hostnames') for alias in LOCALHOST_ALIASES: assert alias in hostnames with pytest.raises(ValueError, match=r'Cannot make asynchronous requests'): cxn.admin_request('users_table.usernames', asynchronous=True) manager.shutdown(connection=cxn)
def test_not_json_serializable(): manager = conftest.Manager(Echo) cxn = connect(**manager.kwargs) e = cxn.link('Echo') with pytest.raises(TypeError, match=r'not JSON serializable'): e.echo(1 + 4j) manager.shutdown(connection=cxn)
def test_wrong_certificate(): manager = conftest.Manager() key = os.path.join(tempfile.gettempdir(), '.msl', 'wrong-certificate.key') cert = os.path.join(tempfile.gettempdir(), '.msl', 'wrong-certificate.crt') assert cryptography.generate_key(path=key) == key assert cryptography.generate_certificate(path=cert, key_path=key) == cert kwargs = manager.kwargs.copy() kwargs['cert_file'] = cert with pytest.raises(ConnectionError) as e: connect(**kwargs) msg = str(e.value) assert 'Perhaps the Network Manager is using a new certificate' in msg assert '{}:{}'.format(constants.HOSTNAME, kwargs['port']) in msg assert 'wrong-certificate.crt' in msg os.remove(key) os.remove(cert) manager.shutdown()
def test_unlink_client_max10(): manager = conftest.Manager(Echo, max_clients=10) clients = [connect(**manager.kwargs) for _ in range(10)] links = [client.link('Echo') for client in clients] for link in links: assert link.service_name == 'Echo' assert link.echo(1, x=2) == [[1], {'x': 2}] cxn = connect(**manager.kwargs) # another Client cannot link with pytest.raises( RuntimeError, match=r'The maximum number of Clients are already linked'): cxn.link('Echo') links[0].unlink() assert links[0]._client is None with pytest.raises( AttributeError, match=r"Cannot access 'echo' since the link has been broken"): links[0].echo(1) # another Client can now link link2 = cxn.link('Echo') assert link2.service_name == 'Echo' assert link2.echo(1, x=2) == [[1], {'x': 2}] link2.unlink() assert link2._client is None with pytest.raises( AttributeError, match=r"Cannot access 'echo' since the link has been broken"): link2.echo(1) manager.shutdown(connection=cxn) # shutting down the manager using cxn will also disconnect all clients for client in clients: assert not client.is_connected()
def shutdown(self, connection=None): # shutdown the Manager and delete the dummy files that were created if connection is None: connection = connect(**self.kwargs) connection.admin_request('shutdown_manager') self._manager_proc.communicate(timeout=5) # self.wait_shutdown(connection.port, '{} will not shutdown'.format(connection)) # for service, thread in self._service_threads.items(): # self.wait_shutdown(service.port, '{} will not shutdown'.format(service)) self.remove_files()
def test_no_certificate(): # calling connect() will automatically get the certificate from the server manager = conftest.Manager(disable_tls=False) cert_file = os.path.join(constants.CERT_DIR, constants.HOSTNAME + '.crt') assert manager.cert_file == cert_file os.remove(manager.cert_file) assert not os.path.isfile(manager.cert_file) assert not os.path.isfile(cert_file) kwargs = manager.kwargs.copy() kwargs['cert_file'] = None kwargs['auto_save'] = True cxn = connect(**kwargs) assert os.path.isfile(cert_file) os.remove(cert_file) manager.shutdown(connection=cxn)
def test_synchronous(): manager = conftest.Manager(Echo) cxn = connect(**manager.kwargs) echo = cxn.link('Echo') # send a request that is ~110 MB args = ['a' * int(1e6), 'b' * int(5e6), 'c' * int(1e7)] kwargs = { '1e6': 'x' * int(1e6), '5e6': 'y' * int(5e6), 'array': list(range(int(1e7))) } reply = echo.echo(*args, **kwargs) assert reply[0] == args assert reply[1] == kwargs manager.shutdown(connection=cxn)
def test_from_service(): # this tests that the Manager can handle multiple # replies from a Service in a single network packet service_connected = [] notifications = [] def create_socket_service(): name = 'ManualService' with socket.socket() as sock: sock.settimeout(5) sock.connect(('localhost', manager.port)) service_connected.append(True) # receive the "username" request request = json.loads(sock.recv(1024).decode()) assert request['attribute'] == 'username' sock.sendall(manager.admin_username.encode() + TERMINATION) # receive the "password" request request = json.loads(sock.recv(1024).decode()) assert request['attribute'] == 'password' sock.sendall(manager.admin_password.encode() + TERMINATION) # receive the "identity" request request = json.loads(sock.recv(1024).decode()) assert request['attribute'] == 'identity' sock.sendall( json.dumps({ 'error': False, 'result': { 'type': 'service', 'name': name, 'attributes': { 'multiple': '' }, }, 'requester': request['requester'], 'uid': request['uid'], }).encode() + TERMINATION) # receive the request from the Client request = json.loads(sock.recv(1024).decode()) response = json.dumps({ 'error': False, 'result': 'the request was a success!', 'requester': request['requester'], 'uid': request['uid'] }).encode() notify1 = json.dumps({ 'error': False, 'result': [[1], { 'a': 1 }], 'service': name, 'uid': NOTIFICATION_UID, }).encode() notify2 = json.dumps({ 'error': False, 'result': [[2], { 'b': 2 }], 'service': name, 'uid': NOTIFICATION_UID, }).encode() sock.sendall(notify1 + TERMINATION + notify2 + TERMINATION + response + TERMINATION) # wait for the Manager to shutdown sock.recv(1024) def handle_notification(*args, **kwargs): notifications.append([args, kwargs]) # start the Manager manager = conftest.Manager(disable_tls=True) # start the Service t = threading.Thread(target=create_socket_service, daemon=True) t.start() while not service_connected: time.sleep(0.1) # perform the test cxn = connect(**manager.kwargs) link = cxn.link('ManualService') link.notification_handler = handle_notification reply = link.multiple() assert reply == 'the request was a success!' assert notifications == [[(1, ), {'a': 1}], [(2, ), {'b': 2}]] manager.shutdown(connection=cxn)
def test_manager_identity(): manager = conftest.Manager(BasicMath, MyArray, Echo) cxn = connect(name='A.B.C', **manager.kwargs) os = '{} {} {}'.format(platform.system(), platform.release(), platform.machine()) language = 'Python ' + platform.python_version() identities = cxn.identities() assert identities['hostname'] == HOSTNAME assert identities['port'] == manager.port assert identities['attributes'] == { 'identity': '() -> dict', 'link': '(service: str) -> bool' } assert identities['language'] == language assert identities['os'] == os assert 'A.B.C[{}:{}]'.format(HOSTNAME, cxn.port) in identities['clients'] assert 'BasicMath' in identities['services'] assert 'Echo' in identities['services'] assert 'MyArray' in identities['services'] identities = cxn.identities(as_string=True) expected = r'''Manager\[{hostname}:\d+] attributes: identity\(\) -> dict link\(service: str\) -> bool language: {language} os: {os} Clients \[1]: A.B.C\[{hostname}:\d+] language: {language} os: {os} Services \[3]: BasicMath\[{hostname}:\d+] attributes: add\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float] divide\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float] ensure_positive\(x:\s?Union\[int, float]\) -> bool euler\(\) -> 2.718281828459045 multiply\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float] pi\(\) -> 3.141592653589793 power\(x:\s?Union\[int, float], n=2\) -> Union\[int, float] set_logging_level\(level:\s?Union\[str, int]\) -> bool subtract\(x:\s?Union\[int, float], y:\s?Union\[int, float]\) -> Union\[int, float] language: {language} max_clients: -1 os: {os} Echo\[{hostname}:\d+] attributes: echo\(\*args, \*\*kwargs\) set_logging_level\(level:\s?Union\[str, int]\) -> bool language: {language} max_clients: -1 os: {os} MyArray\[{hostname}:\d+] attributes: linspace\(start:\s?Union\[int, float], stop:\s?Union\[int, float], n=100\) -> List\[float] scalar_multiply\(scalar:\s?Union\[int, float], data:\s?List\[float]\) -> List\[float] set_logging_level\(level:\s?Union\[str, int]\) -> bool language: {language} max_clients: -1 os: {os} '''.format(hostname=HOSTNAME, language=language, os=os).splitlines() id_lines = identities.splitlines() assert len(id_lines) == len(expected) for pattern, string in zip(expected, id_lines): assert re.match(pattern, string) manager.shutdown(connection=cxn)
def __init__(self, *service_classes, disable_tls=False, password_manager=None, auth_login=True, auth_hostname=False, cert_common_name=None, add_heartbeat_task=False, read_limit=None, **kwargs): """Starts the Network Manager and all specified Services to use for testing. Parameters ---------- service_classes The Service sub-classes to start (they have NOT been instantiated). **kwargs These are all sent to Service.__init__ for all `service_classes`. """ self.port = self.get_available_port() self.auth_hostname = auth_hostname self.auth_login = auth_login self.disable_tls = disable_tls self._manager_proc = None self.remove_files() key_pw = 'dummy pw!' # use a password for the key.. just for fun cryptography.generate_key(path=self.key_file, password=key_pw, algorithm='ecc') cryptography.generate_certificate(path=self.cert_file, key_path=self.key_file, key_password=key_pw, name=cert_common_name) # need a UsersTable with an administrator to be able to shutdown the Manager ut = UsersTable(database=self.database) self.admin_username, self.admin_password = '******', 'whatever' ut.insert(self.admin_username, self.admin_password, True) ut.close() # a convenience dictionary for connecting to the Manager as a Service or a Client self.kwargs = { 'username': self.admin_username, 'password': self.admin_password, 'port': self.port, 'cert_file': self.cert_file, 'disable_tls': disable_tls, 'password_manager': password_manager, 'read_limit': read_limit, } # start the Network Manager in a subprocess command = [ sys.executable, '-c', 'from msl.network import cli; cli.main()', 'start', '-p', str(self.port), '-c', self.cert_file, '-k', self.key_file, '-D', key_pw, '-d', self.database, '-l', self.log_file ] if disable_tls: command.append('--disable-tls') if password_manager: command.extend(['-P', password_manager]) elif auth_hostname: command.append('--auth-hostname') elif auth_login: command.append('--auth-login') # start the Network Manager self._manager_proc = subprocess.Popen(command) self.wait_start(self.port, 'Cannot start Manager') # start all Services self._service_threads = {} if service_classes: cxn = connect(**self.kwargs) for cls in service_classes: name = cls.__name__ service = cls(**kwargs) if add_heartbeat_task and name == 'Heartbeat': service.add_tasks(service.emit()) thread = Thread(target=service.start, kwargs=self.kwargs, daemon=True) thread.start() t0 = time.time() while True: time.sleep(0.1) services = cxn.identities()['services'] if name in services: break if time.time() - t0 > 30: in_use = self.is_port_in_use(service.port) self.shutdown(cxn) raise RuntimeError('Cannot start {0} service.\n' 'Is Service port in use? {1}\n' '{0}.start kwargs: {2}\n' 'Services: {3}'.format( name, in_use, self.kwargs, services)) self._service_threads[service] = thread cxn.disconnect()
def test_multiple_services(): class ShutdownableService(Service): def shutdown_service(self, *args, **kwargs): return args, kwargs class AddService(ShutdownableService): def add(self, a, b): return a + b class SubtractService(ShutdownableService): def subtract(self, a, b): return a - b password_manager = '<hey~you>' port = conftest.Manager.get_available_port() run_thread = threading.Thread( target=run_services, args=(AddService(), SubtractService()), kwargs={ 'password_manager': password_manager, 'port': port, 'log_file': conftest.Manager.log_file } ) run_thread.start() # wait for the Manager to be running conftest.Manager.wait_start(port, 'Cannot connect to manager') cxn = connect(password_manager=password_manager, port=port) # wait for the Services to be running before linking while len(cxn.identities()['services']) != 2: time.sleep(0.1) s = cxn.link('SubtractService') a = cxn.link('AddService') assert a.add(1, 2) == 3 assert s.subtract(1, 2) == -1 # shut down the AddService reply = a.shutdown_service(1, x=9) assert reply == [[1], {'x': 9}] while len(cxn.identities()['services']) == 2: time.sleep(0.1) with pytest.raises(AttributeError, match=r'the link has been broken'): assert a.add(1, 2) is None with pytest.raises(RuntimeError): cxn.link('AddService') # the SubtractService is still available assert s.subtract(9, 4) == 5 # shut down the SubtractService reply = s.shutdown_service('foo', 'bar', xyz=None) assert reply == [['foo', 'bar'], {'xyz': None}] with pytest.raises(AttributeError, match=r'the link has been broken'): s.subtract(1, 2) # the `run_services` function will block the unittests forever if the # Client did not shutdown both Services run_thread.join()
def test_no_manager_no_timeout_remotehost(): host = 'MSLNZ12345' match = r'Cannot connect to {}:{}$'.format(host, constants.PORT) with pytest.raises(ConnectionError, match=match): connect(host=host, disable_tls=True, timeout=None)
def test_no_manager_no_timeout_localhost(): port = conftest.Manager.get_available_port() match = r'Cannot connect to {}:{}$'.format(constants.HOSTNAME, port) with pytest.raises(ConnectionError, match=match): connect(port=port, disable_tls=True, timeout=None)
def test_no_manager_no_certificate_localhost(): conftest.Manager.remove_files() match = r'Make sure a Manager is running on this computer$' with pytest.raises(ConnectionError, match=match): connect(disable_tls=False)