def test_get_servers_from_scope_one_server(self, mock_get_now, mock_app): mock_get_now.return_value = now s0 = Server(id='00000000-0000-0000-0000-000000000000', name='node0', port=5000, me=True, created_on=old_age) db.session.add(s0) mock_app.dm.cluster_manager.get_alive.return_value = [s0.id] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) s0.created_on = now db.session.flush() quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) mock_app.dm.cluster_manager.get_alive.return_value = [] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([s0], quorum) s0.l_ignore_on_lock = True db.session.flush() quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual([], quorum)
def setUp(self): """Create and configure a new self.app instance for each test.""" # create a temporary file to isolate the database for each test # create the self.app with common test config self.app = Flask(__name__) @self.app.route('/', methods=['GET', 'POST']) @forward_or_dispatch() def hello(): return {'msg': 'default response'} self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() db.init_app(self.app) db.create_all() self.srv1 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb1', name='server1', dns_or_ip='192.168.1.9', port=7123) Route(self.srv1, cost=0) self.srv2 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb2', name='server2', dns_or_ip='192.168.1.10', port=7124) Route(self.srv2, self.srv1, cost=1) db.session.add_all([self.srv1, self.srv2]) db.session.commit()
def _notify_cluster_out(self): with self.dm.flask_app.app_context(): servers = Server.get_neighbours() if servers: self.logger.debug( f"Sending shutdown to {', '.join([s.name for s in servers])}" ) else: self.logger.debug("No server to send shutdown information") if servers: responses = asyncio.run( ntwrk.parallel_requests( servers, 'post', view_or_url='api_1_0.cluster_out', view_data=dict(server_id=str(Server.get_current().id)), json={ 'death': get_now().strftime(defaults.DATEMARK_FORMAT) }, timeout=2, auth=get_root_auth())) if self.logger.level <= logging.DEBUG: for r in responses: if not r.ok: self.logger.warning( f"Unable to send data to {r.server}: {r}")
def test_get_servers_from_scope_less_than_min_quorum( self, mock_get_now, mock_app): mock_get_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc) me = Server(id='00000000-0000-0000-0000-000000000000', name='node0', port=5000, me=True, created_on=now) db.session.add(me) servers = [] for i in range(1, 4): s = Server(f'node{i}', port=5000, created_on=old_age) r = Route(s, cost=0) db.session.add_all([s, r]) servers.append(s) mock_app.dm.cluster_manager.get_alive.return_value = [ s.id for s in servers ] quorum = get_servers_from_scope(scope=Scope.CATALOG) self.assertEqual(4, len(quorum)) self.assertIn(Server.get_current(), quorum)
def fill_database(self, node): db.create_all() d = Dimension.from_json(self.dim) d.current = True s1 = Server(id='00000000-0000-0000-0000-000000000001', name='node1', created_on=now, me=node == 'node1') g11 = Gate(id='00000000-0000-0000-0000-000000000011', server=s1, port=5000, dns=s1.name) s2 = Server(id='00000000-0000-0000-0000-000000000002', name='node2', created_on=now, me=node == 'node2') g12 = Gate(id='00000000-0000-0000-0000-000000000012', server=s2, port=5000, dns=s2.name) s3 = Server(id='00000000-0000-0000-0000-000000000003', name='node3', created_on=now, me=node == 'node3') g13 = Gate(id='00000000-0000-0000-0000-000000000013', server=s3, port=5000, dns=s3.name) s4 = Server(id='00000000-0000-0000-0000-000000000004', name='node4', created_on=now, me=node == 'node4') g14 = Gate(id='00000000-0000-0000-0000-000000000014', server=s4, port=5000, dns=s4.name) if node == 'node1': self.s1 = s1 self.s2 = s2 self.s3 = s3 self.s4 = s4 Route(s2, g12) Route(s3, s2, 1) elif node == 'node2': Route(s1, g11) Route(s3, g13) elif node == 'node3': Route(s1, s2, 1) Route(s2, g12) db.session.add_all([d, s1, s2, s3, s4]) db.session.commit()
def test_get_neighbours_no_route(self): n1 = Server('n1', port=8000) me = Server('me', port=8000, me=True) db.session.add_all([n1, me]) self.assertListEqual([], me.get_neighbours())
def fill_database(self): self.dest1 = Server('dest1', port=8000) self.dest2 = Server('dest2', port=8000) self.log = Log(source_server=self.s1, target='/var/log/log1.log', destination_server=self.dest1) db.session.add_all([self.dest1, self.dest2, self.log]) db.session.commit()
def fill_database(self): """Create and configure a new app instance for each test.""" self.dest1 = Server('dest1', port=8000) self.dest2 = Server('dest2', port=8000) self.log = Log(source_server=self.s1, target='/var/log/log1.log', destination_server=self.dest1) db.session.add_all([self.dest1, self.dest2, self.log])
def test_execute_send_software(self, mock_post, mock_get): at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001') soft = Software(name='test', version=1, filename='test.zip') node1 = Server('nodeA', port=5000) node2 = Server('nodeB', port=5000) ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/') ssa2 = SoftwareServerAssociation(software=soft, server=node2, path='/') ssa3 = SoftwareServerAssociation(software=soft, server=self.s1, path='/') db.session.add_all([soft, node1, node2, ssa1, ssa2, ssa3]) mock_post.return_value = Response(msg={'transfer_id': 1}, code=at.expected_rc) mock_get.return_value = Response(msg={ "route_list": [{ "cost": 0, "destination_id": f"{node1.id}", }, { "cost": 1, "destination_id": f"{self.s1.id}", }], }, code=200, server=node2) ro = NativeSoftwareSendOperation(code, expected_stdout=at.expected_stdout, expected_stderr=at.expected_stderr, expected_rc=at.expected_rc) cp = ro._execute(dict(input=dict(software=soft.id, server=node2.id, dest_path='dest', chunk_size=20, max_senders=2)), timeout=None, context=self.context) mock_post.assert_called_once_with(node1, 'api_1_0.send', json=dict(software_id=str(soft.id), dest_server_id=str( node2.id), background=False, include_transfer_data=True, force=True, dest_path='dest', chunk_size=20, max_senders=2), timeout=None, identity=ROOT) self.assertTrue(cp.success) self.assertEqual(flask.json.dumps(mock_post.return_value.msg), cp.stdout)
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config self.app = create_app('test') self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() self.auth = HTTPBearerAuth(create_access_token('00000000-0000-0000-0000-000000000001')) db.create_all() self.src = Server('source', port=5000) self.dst = Server('destination', port=5000)
def setUp(self): super().setUp() set_initial() self.n1 = Server("node1", port=8000) Route(self.n1, cost=0) self.n2 = Server("node2", port=8000) Route(self.n2, cost=0) db.session.add_all([self.n1, self.n2]) db.session.commit() self.datemark = Catalog.max_catalog(str)
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config User.set_initial() self.source = Server('source', port=8000, me=True) self.dest = Server('dest', port=8000) Route(self.dest, cost=0) db.session.add_all([self.source, self.dest]) _PygtailBuffer._fh = mock.MagicMock() self.mock_dm = mock.Mock() self.log_sender = LogSender(dimensigon=self.mock_dm)
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config self.app = create_app('test') self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() self.auth = HTTPBearerAuth(create_access_token('00000000-0000-0000-0000-000000000001')) db.create_all() self.me = Server('me', port=5000, _me=True) self.remote = Server('remote', port=5000) db.session.add_all([self.remote, self.me]) self.maxDiff = None
def _fill_database(self, app: Flask): with mock.patch('dimensigon.domain.entities.get_now') as mock_get_now: mock_get_now.return_value = defaults.INITIAL_DATEMARK node = app.config['SERVER_NAME'] with app.app_context(): db.create_all() event.listen(db.session, 'after_commit', receive_after_commit) set_initial(**self.initials) d = Dimension.from_json(self.dim) d.current = True s1 = Server('node1', created_on=defaults.INITIAL_DATEMARK, id=self.SERVER1, me=node == 'node1') g11 = Gate(id='00000000-0000-0000-0000-000000000011', server=s1, port=5000, dns=s1.name) s2 = Server('node2', created_on=defaults.INITIAL_DATEMARK, id=self.SERVER2, me=node == 'node2') g12 = Gate(id='00000000-0000-0000-0000-000000000012', server=s2, port=5000, dns=s2.name) s3 = Server('node3', created_on=defaults.INITIAL_DATEMARK, id=self.SERVER3, me=node == 'node3') g13 = Gate(id='00000000-0000-0000-0000-000000000013', server=s3, port=5000, dns=s3.name) if node == 'node1': Route(s2, g12) Route(s3, g13) elif node == 'node2': Route(s1, g11) Route(s3, g13) elif node == 'node3': Route(s1, g11) Route(s2, g12) self.fill_database() db.session.add_all([d, s1, s2, s3]) db.session.commit() if node == 'node1': self.s1, self.s2, self.s3 = db.session.merge( s1), db.session.merge(s2), db.session.merge(s3)
def healthcheck(): if request.method == 'POST' and isinstance(g.source, Server): data = request.get_json() try: heartbeat = dt.datetime.strptime(data['heartbeat'], defaults.DATETIME_FORMAT) except: raise errors.InvalidDateFormat(data['heartbeat'], defaults.DATETIME_FORMAT) current_app.dm.cluster_manager.put(data['me'], heartbeat) catalog_ver = Catalog.max_catalog() data = { "version": dimensigon.__version__, "catalog_version": catalog_ver.strftime(defaults.DATEMARK_FORMAT) if catalog_ver else None, "services": [], } if not check_param_in_uri('human'): server = {'id': str(g.server.id), 'name': g.server.name} neighbours = [{ 'id': str(s.id), 'name': s.name } for s in Server.get_neighbours()] cluster = { 'alive': current_app.dm.cluster_manager.get_alive(), 'in_coma': current_app.dm.cluster_manager.get_zombies() } else: server = g.server.name neighbours = sorted([s.name for s in Server.get_neighbours()]) cluster = { 'alive': sorted([ getattr(Server.query.get(i), 'name', i) for i in current_app.dm.cluster_manager.get_alive() ]), 'in_coma': sorted([ getattr(Server.query.get(i), 'name', i) for i in current_app.dm.cluster_manager.get_zombies() ]) } data.update(server=server, neighbours=neighbours, cluster=cluster, now=get_now().strftime(defaults.DATETIME_FORMAT)) return data
def setUp(self, mock_now): """Create and configure a new app instance for each test.""" mock_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc) # create the app with common test config self.app = create_app('test') self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() db.create_all() Locker.set_initial() Server.set_initial() db.session.commit()
def test_to_json_proxy_remote(self): dest = Server('dest', port=8000) proxy = Server('proxy', port=8000) r = Route(destination=dest, proxy_server_or_gate=proxy, cost=1) db.session.add_all([dest, proxy]) db.session.commit() self.assertDictEqual( { 'destination_id': str(dest.id), 'gate_id': None, 'proxy_server_id': str(proxy.id), 'cost': 1 }, r.to_json())
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config self.app = create_app('test') self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() self.headers = { "Authorization": f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}" } db.create_all() set_initial() self.soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file') self.soft2 = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa2', name='test', version='2', filename='file') self.ssa = SoftwareServerAssociation(software=self.soft, server=Server.get_current(), path='/root') db.session.add_all([self.soft, self.soft2, self.ssa]) db.session.commit()
def setUp(self): self.maxDiff = None self.app_join = create_app('test') self.app_join.config['SERVER_NAME'] = 'join' self.app_join_context = self.app_join.app_context() self.client_join = self.app_join.test_client() with self.app_join.app_context(): db.create_all() set_initial(server=False) self.join_server_id = Server.set_initial() db.session.commit() self.mock_dm = mock.MagicMock() self.mock_dm.flask_app = self.app_join self.mock_dm.engine = db.engine self.mock_dm.catalog_manager.db_update_catalog = update_db_catalog super().setUp() self.app.config['SECURIZER'] = True self.app_join.config['SECURIZER'] = True self.token = create_access_token(JOIN, additional_claims={'applicant': 'me'}) self.setUpPyfakefs() open('/origin_key', 'w').write('keyfile') open('/origin_cert', 'w').write('certfile')
def setUp(self): """Create and configure a new self.app instance for each test.""" # create a temporary file to isolate the database for each test # create the self.app with common test config self.app = Flask(__name__) self.app.config['JWT_SECRET_KEY'] = 'super-secret' self.jwt = JWTManager(self.app) @self.app.route('/', methods=['GET']) @jwt_required() @lock_catalog def hello(): return {'msg': 'default response'} self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() db.init_app(self.app) db.create_all() # self.d = generate_dimension('test') self.srv1 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb1', name='server1', me=True) Locker.set_initial() db.session.add(self.srv1) db.session.commit()
async def _async_set_current_neighbours( neighbours: t.List[Server] = None, changed_routes: t.Dict[Server, RouteContainer] = None) -> t.List[Server]: """Function checks and sets neighbours Args: neighbours: list of neighbours changed_routes: reference to a dict which will be populated with new routes Returns: list of servers which are not neighbours anymore """ not_neighbours_anymore = [] if neighbours is None: neighbours = Server.get_neighbours() if neighbours: resp = await asyncio.gather( *[async_check_gates(server) for server in neighbours]) for route, server in zip(resp, neighbours): if isinstance(route, RouteContainer): server.set_route(route) if changed_routes is not None: changed_routes[server] = route elif route is None: not_neighbours_anymore.append(server) rc = RouteContainer(None, None, None) server.set_route(rc) if changed_routes is not None: changed_routes[server] = rc return not_neighbours_anymore
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config self.app = create_app('test') self.app.config['SECURIZER'] = True @self.app.route('/', methods=['GET', 'POST']) @securizer def home(): return {'msg': 'default response'} self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() db.init_app(self.app) db.create_all() self.server = Server('me', port=5000, me=True) db.session.add(self.server) self.dim = generate_dimension('dimension') self.dim.current = True db.session.add(self.dim) self.token = create_access_token('test') self.auth = HTTPBearerAuth(self.token) self.url = 'https://me:5000/'
def setUp(self, mocked_now): self.initials = dict(self.initials) self.initials.update(action_template=False) mocked_now.return_value = now1 super().setUp() with self.app2_context: mocked_now.return_value = now2 soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file') at = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2', name='mkdir', version=1, action_type=ActionType.SHELL, code='mkdir {dir}') db.session.add_all([soft, at]) db.session.commit() mocked_now.return_value = now3 ssa = SoftwareServerAssociation(software=soft, server=Server.get_current(), path='/root') db.session.add(ssa) db.session.commit() self.soft_json = soft.to_json() self.at_json = at.to_json() self.catalog = fetch_catalog(now1) self.mock_queue = mock.Mock() self.mock_dm = mock.Mock() self.mock_dm.flask_app = self.app self.mock_dm.engine = db.engine self.mock_dm.manager.dict.return_value = dict() self.mock_dm.server_id = self.s1.id self.cm = CatalogManager("Catalog", startup_event=threading.Event(), shutdown_event=threading.Event(), publish_q=self.mock_queue, event_q=None, dimensigon=self.mock_dm) db.session.commit()
def test_execute_send_software_error(self, mock_post, mock_get): at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001') soft = Software(name='test', version=1, filename='test.zip') node1 = Server('nodeA', port=5000) ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/') db.session.add_all([soft, node1, ssa1]) mock_post.return_value = Response(msg={'error': 'message'}, code=400) mock_get.return_value = Response(code=400) ro = NativeSoftwareSendOperation(code, expected_stdout=at.expected_stdout, expected_stderr=at.expected_stderr, expected_rc=at.expected_rc) cp = ro._execute( dict(input=dict(software=str(soft.id), server=str(node1.id))), timeout=10, context=self.context) mock_post.assert_called_once_with(node1, 'api_1_0.send', json=dict(software_id=str(soft.id), dest_server_id=str( node1.id), background=False, include_transfer_data=True, force=True), timeout=10, identity=ROOT) self.assertFalse(cp.success) self.assertEqual(flask.json.dumps(mock_post.return_value.msg), cp.stdout)
def test_get_internal_error_server(self, m): msg = '<html>Iternal error server</html>' status = 500 responses.add(responses.GET, self.url, status=status, body=msg) m.get(self.url, status=status, body=msg) resp = get(Server.get_current(), 'home') self.assertEqual(status, resp.code) self.assertEqual(msg, resp.msg) resp = run(async_get(Server.get_current(), 'home')) self.assertEqual(status, resp.code) self.assertEqual(msg, resp.msg)
def _proxy_request(request: 'flask.Request', destination: Server, verify=False) -> requests.Response: url = destination.url() + request.full_path req_data = request.get_json() if request.path == '/ping': server_data = {'id': str(g.server.id), 'name': g.server.name, 'time': get_now().strftime(defaults.DATETIME_FORMAT)} if req_data: if 'servers' not in req_data: req_data['servers'] = {} req_data['servers'].update({len(req_data['servers']) + 1: server_data}) else: req_data = dict(servers={1: server_data}) kwargs = { 'json': req_data, 'allow_redirects': False } headers = {key.lower(): value for key, value in request.headers.items()} # Let requests reset the host for us. if 'host' in headers: del headers['host'] headers['d-source'] = headers.get('d-source', '') + ':' + str(g.server.id) kwargs['headers'] = headers cookies = request.cookies kwargs['cookies'] = cookies return requests.request(request.method, url, verify=verify, **kwargs)
def cluster_in(server_id): user = User.get_current() data = request.get_json() if user and user.name == 'root': try: keepalive = dt.datetime.strptime(data.get('keepalive'), defaults.DATEMARK_FORMAT) except ValueError: raise errors.InvalidDateFormat(data.get('keepalive'), defaults.DATEMARK_FORMAT) current_app.dm.cluster_manager.put(server_id, keepalive) _cluster_logger.debug( f"{getattr(Server.query.get(server_id), 'name', server_id) or server_id} is a new alive server" ) current_app.dm.route_manager.new_node_in_cluster( server_id, data['routes']) return { 'cluster': current_app.dm.cluster_manager.get_cluster( defaults.DATEMARK_FORMAT), 'neighbours': [s.id for s in Server.get_neighbours()] }, 200 else: raise errors.UserForbiddenError
def setUp(self): """Create and configure a new app instance for each test.""" # create the app with common test config self.app = create_app('test') self.app_context = self.app.app_context() self.app_context.push() self.client = self.app.test_client() self.headers = { "Authorization": f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}" } db.create_all() # set_initial(server=False) self.srv1 = Server('node1', id='00000000-0000-0000-0000-000000000001', me=True) self.file = File(source_server=self.srv1, target='/etc/ssh/sshd_config', id='00000000-0000-0000-0000-000000000002') self.fsa = FileServerAssociation(file=self.file, destination_server=self.srv1) db.session.add_all([self.srv1, self.file, self.fsa]) db.session.commit()
def load_global_data_into_context(): from dimensigon.domain.entities import Server, Dimension from dimensigon.web.decorators import set_source global _dimension, _server set_source() g.server = Server.get_current() g.dimension = Dimension.get_current()
def set_servers_and_routes(self): self.n1 = Server(name='n1', dns_or_ip='1.1.1.1') self.n2 = Server(name='n2', dns_or_ip='n2_dns') self.n3 = Server(name='n3', port=8000) self.r1 = Server(name='r1', dns_or_ip='3.3.3.3') self.r2 = Server(name='r2', dns_or_ip='r2_dns') db.session.add_all([self.n1, self.n2, self.r1, self.r2]) db.session.commit() Route(destination=self.n1, cost=0) Route(destination=self.n2, cost=0) Route(destination=self.n3, cost=0) Route(destination=self.r1, proxy_server_or_gate=self.n1, cost=1) Route(destination=self.r2, proxy_server_or_gate=self.n2, cost=1) db.session.commit()