def test_get_servers_from_scope_one_server(self, mock_get_now, mock_app):
mock_get_now.return_value = now
s0 = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=old_age)
db.session.add(s0)
mock_app.dm.cluster_manager.get_alive.return_value = [s0.id]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
s0.created_on = now
db.session.flush()
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
mock_app.dm.cluster_manager.get_alive.return_value = []
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([s0], quorum)
s0.l_ignore_on_lock = True
db.session.flush()
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual([], quorum)
def setUp(self):
"""Create and configure a new self.app instance for each test."""
# create a temporary file to isolate the database for each test
# create the self.app with common test config
self.app = Flask(__name__)
@self.app.route('/', methods=['GET', 'POST'])
@forward_or_dispatch()
def hello():
return {'msg': 'default response'}
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
db.init_app(self.app)
db.create_all()
self.srv1 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb1', name='server1',
dns_or_ip='192.168.1.9', port=7123)
Route(self.srv1, cost=0)
self.srv2 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb2', name='server2',
dns_or_ip='192.168.1.10', port=7124)
Route(self.srv2, self.srv1, cost=1)
db.session.add_all([self.srv1, self.srv2])
db.session.commit()
def _notify_cluster_out(self):
with self.dm.flask_app.app_context():
servers = Server.get_neighbours()
if servers:
self.logger.debug(
f"Sending shutdown to {', '.join([s.name for s in servers])}"
)
else:
self.logger.debug("No server to send shutdown information")
if servers:
responses = asyncio.run(
ntwrk.parallel_requests(
servers,
'post',
view_or_url='api_1_0.cluster_out',
view_data=dict(server_id=str(Server.get_current().id)),
json={
'death':
get_now().strftime(defaults.DATEMARK_FORMAT)
},
timeout=2,
auth=get_root_auth()))
if self.logger.level <= logging.DEBUG:
for r in responses:
if not r.ok:
self.logger.warning(
f"Unable to send data to {r.server}: {r}")
def test_get_servers_from_scope_less_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
me = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=now)
db.session.add(me)
servers = []
for i in range(1, 4):
s = Server(f'node{i}', port=5000, created_on=old_age)
r = Route(s, cost=0)
db.session.add_all([s, r])
servers.append(s)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(4, len(quorum))
self.assertIn(Server.get_current(), quorum)
def fill_database(self, node):
db.create_all()
d = Dimension.from_json(self.dim)
d.current = True
s1 = Server(id='00000000-0000-0000-0000-000000000001', name='node1', created_on=now, me=node == 'node1')
g11 = Gate(id='00000000-0000-0000-0000-000000000011', server=s1, port=5000, dns=s1.name)
s2 = Server(id='00000000-0000-0000-0000-000000000002', name='node2', created_on=now, me=node == 'node2')
g12 = Gate(id='00000000-0000-0000-0000-000000000012', server=s2, port=5000, dns=s2.name)
s3 = Server(id='00000000-0000-0000-0000-000000000003', name='node3', created_on=now, me=node == 'node3')
g13 = Gate(id='00000000-0000-0000-0000-000000000013', server=s3, port=5000, dns=s3.name)
s4 = Server(id='00000000-0000-0000-0000-000000000004', name='node4', created_on=now, me=node == 'node4')
g14 = Gate(id='00000000-0000-0000-0000-000000000014', server=s4, port=5000, dns=s4.name)
if node == 'node1':
self.s1 = s1
self.s2 = s2
self.s3 = s3
self.s4 = s4
Route(s2, g12)
Route(s3, s2, 1)
elif node == 'node2':
Route(s1, g11)
Route(s3, g13)
elif node == 'node3':
Route(s1, s2, 1)
Route(s2, g12)
db.session.add_all([d, s1, s2, s3, s4])
db.session.commit()
def test_get_neighbours_no_route(self):
n1 = Server('n1', port=8000)
me = Server('me', port=8000, me=True)
db.session.add_all([n1, me])
self.assertListEqual([], me.get_neighbours())
def fill_database(self):
self.dest1 = Server('dest1', port=8000)
self.dest2 = Server('dest2', port=8000)
self.log = Log(source_server=self.s1,
target='/var/log/log1.log',
destination_server=self.dest1)
db.session.add_all([self.dest1, self.dest2, self.log])
db.session.commit()
def fill_database(self):
"""Create and configure a new app instance for each test."""
self.dest1 = Server('dest1', port=8000)
self.dest2 = Server('dest2', port=8000)
self.log = Log(source_server=self.s1,
target='/var/log/log1.log',
destination_server=self.dest1)
db.session.add_all([self.dest1, self.dest2, self.log])
def test_execute_send_software(self, mock_post, mock_get):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version=1, filename='test.zip')
node1 = Server('nodeA', port=5000)
node2 = Server('nodeB', port=5000)
ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/')
ssa2 = SoftwareServerAssociation(software=soft, server=node2, path='/')
ssa3 = SoftwareServerAssociation(software=soft,
server=self.s1,
path='/')
db.session.add_all([soft, node1, node2, ssa1, ssa2, ssa3])
mock_post.return_value = Response(msg={'transfer_id': 1},
code=at.expected_rc)
mock_get.return_value = Response(msg={
"route_list": [{
"cost": 0,
"destination_id": f"{node1.id}",
}, {
"cost": 1,
"destination_id": f"{self.s1.id}",
}],
},
code=200,
server=node2)
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(dict(input=dict(software=soft.id,
server=node2.id,
dest_path='dest',
chunk_size=20,
max_senders=2)),
timeout=None,
context=self.context)
mock_post.assert_called_once_with(node1,
'api_1_0.send',
json=dict(software_id=str(soft.id),
dest_server_id=str(
node2.id),
background=False,
include_transfer_data=True,
force=True,
dest_path='dest',
chunk_size=20,
max_senders=2),
timeout=None,
identity=ROOT)
self.assertTrue(cp.success)
self.assertEqual(flask.json.dumps(mock_post.return_value.msg),
cp.stdout)
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.auth = HTTPBearerAuth(create_access_token('00000000-0000-0000-0000-000000000001'))
db.create_all()
self.src = Server('source', port=5000)
self.dst = Server('destination', port=5000)
def setUp(self):
super().setUp()
set_initial()
self.n1 = Server("node1", port=8000)
Route(self.n1, cost=0)
self.n2 = Server("node2", port=8000)
Route(self.n2, cost=0)
db.session.add_all([self.n1, self.n2])
db.session.commit()
self.datemark = Catalog.max_catalog(str)
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
User.set_initial()
self.source = Server('source', port=8000, me=True)
self.dest = Server('dest', port=8000)
Route(self.dest, cost=0)
db.session.add_all([self.source, self.dest])
_PygtailBuffer._fh = mock.MagicMock()
self.mock_dm = mock.Mock()
self.log_sender = LogSender(dimensigon=self.mock_dm)
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.auth = HTTPBearerAuth(create_access_token('00000000-0000-0000-0000-000000000001'))
db.create_all()
self.me = Server('me', port=5000, _me=True)
self.remote = Server('remote', port=5000)
db.session.add_all([self.remote, self.me])
self.maxDiff = None
def _fill_database(self, app: Flask):
with mock.patch('dimensigon.domain.entities.get_now') as mock_get_now:
mock_get_now.return_value = defaults.INITIAL_DATEMARK
node = app.config['SERVER_NAME']
with app.app_context():
db.create_all()
event.listen(db.session, 'after_commit', receive_after_commit)
set_initial(**self.initials)
d = Dimension.from_json(self.dim)
d.current = True
s1 = Server('node1',
created_on=defaults.INITIAL_DATEMARK,
id=self.SERVER1,
me=node == 'node1')
g11 = Gate(id='00000000-0000-0000-0000-000000000011',
server=s1,
port=5000,
dns=s1.name)
s2 = Server('node2',
created_on=defaults.INITIAL_DATEMARK,
id=self.SERVER2,
me=node == 'node2')
g12 = Gate(id='00000000-0000-0000-0000-000000000012',
server=s2,
port=5000,
dns=s2.name)
s3 = Server('node3',
created_on=defaults.INITIAL_DATEMARK,
id=self.SERVER3,
me=node == 'node3')
g13 = Gate(id='00000000-0000-0000-0000-000000000013',
server=s3,
port=5000,
dns=s3.name)
if node == 'node1':
Route(s2, g12)
Route(s3, g13)
elif node == 'node2':
Route(s1, g11)
Route(s3, g13)
elif node == 'node3':
Route(s1, g11)
Route(s2, g12)
self.fill_database()
db.session.add_all([d, s1, s2, s3])
db.session.commit()
if node == 'node1':
self.s1, self.s2, self.s3 = db.session.merge(
s1), db.session.merge(s2), db.session.merge(s3)
def healthcheck():
if request.method == 'POST' and isinstance(g.source, Server):
data = request.get_json()
try:
heartbeat = dt.datetime.strptime(data['heartbeat'],
defaults.DATETIME_FORMAT)
except:
raise errors.InvalidDateFormat(data['heartbeat'],
defaults.DATETIME_FORMAT)
current_app.dm.cluster_manager.put(data['me'], heartbeat)
catalog_ver = Catalog.max_catalog()
data = {
"version":
dimensigon.__version__,
"catalog_version":
catalog_ver.strftime(defaults.DATEMARK_FORMAT)
if catalog_ver else None,
"services": [],
}
if not check_param_in_uri('human'):
server = {'id': str(g.server.id), 'name': g.server.name}
neighbours = [{
'id': str(s.id),
'name': s.name
} for s in Server.get_neighbours()]
cluster = {
'alive': current_app.dm.cluster_manager.get_alive(),
'in_coma': current_app.dm.cluster_manager.get_zombies()
}
else:
server = g.server.name
neighbours = sorted([s.name for s in Server.get_neighbours()])
cluster = {
'alive':
sorted([
getattr(Server.query.get(i), 'name', i)
for i in current_app.dm.cluster_manager.get_alive()
]),
'in_coma':
sorted([
getattr(Server.query.get(i), 'name', i)
for i in current_app.dm.cluster_manager.get_zombies()
])
}
data.update(server=server,
neighbours=neighbours,
cluster=cluster,
now=get_now().strftime(defaults.DATETIME_FORMAT))
return data
def setUp(self, mock_now):
"""Create and configure a new app instance for each test."""
mock_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc)
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
db.create_all()
Locker.set_initial()
Server.set_initial()
db.session.commit()
def test_to_json_proxy_remote(self):
dest = Server('dest', port=8000)
proxy = Server('proxy', port=8000)
r = Route(destination=dest, proxy_server_or_gate=proxy, cost=1)
db.session.add_all([dest, proxy])
db.session.commit()
self.assertDictEqual(
{
'destination_id': str(dest.id),
'gate_id': None,
'proxy_server_id': str(proxy.id),
'cost': 1
}, r.to_json())
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.headers = {
"Authorization":
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
db.create_all()
set_initial()
self.soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1',
name='test',
version='1',
filename='file')
self.soft2 = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa2',
name='test',
version='2',
filename='file')
self.ssa = SoftwareServerAssociation(software=self.soft,
server=Server.get_current(),
path='/root')
db.session.add_all([self.soft, self.soft2, self.ssa])
db.session.commit()
def setUp(self):
self.maxDiff = None
self.app_join = create_app('test')
self.app_join.config['SERVER_NAME'] = 'join'
self.app_join_context = self.app_join.app_context()
self.client_join = self.app_join.test_client()
with self.app_join.app_context():
db.create_all()
set_initial(server=False)
self.join_server_id = Server.set_initial()
db.session.commit()
self.mock_dm = mock.MagicMock()
self.mock_dm.flask_app = self.app_join
self.mock_dm.engine = db.engine
self.mock_dm.catalog_manager.db_update_catalog = update_db_catalog
super().setUp()
self.app.config['SECURIZER'] = True
self.app_join.config['SECURIZER'] = True
self.token = create_access_token(JOIN,
additional_claims={'applicant': 'me'})
self.setUpPyfakefs()
open('/origin_key', 'w').write('keyfile')
open('/origin_cert', 'w').write('certfile')
def setUp(self):
"""Create and configure a new self.app instance for each test."""
# create a temporary file to isolate the database for each test
# create the self.app with common test config
self.app = Flask(__name__)
self.app.config['JWT_SECRET_KEY'] = 'super-secret'
self.jwt = JWTManager(self.app)
@self.app.route('/', methods=['GET'])
@jwt_required()
@lock_catalog
def hello():
return {'msg': 'default response'}
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
db.init_app(self.app)
db.create_all()
# self.d = generate_dimension('test')
self.srv1 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb1', name='server1', me=True)
Locker.set_initial()
db.session.add(self.srv1)
db.session.commit()
async def _async_set_current_neighbours(
neighbours: t.List[Server] = None,
changed_routes: t.Dict[Server,
RouteContainer] = None) -> t.List[Server]:
"""Function checks and sets neighbours
Args:
neighbours: list of neighbours
changed_routes: reference to a dict which will be populated with new routes
Returns:
list of servers which are not neighbours anymore
"""
not_neighbours_anymore = []
if neighbours is None:
neighbours = Server.get_neighbours()
if neighbours:
resp = await asyncio.gather(
*[async_check_gates(server) for server in neighbours])
for route, server in zip(resp, neighbours):
if isinstance(route, RouteContainer):
server.set_route(route)
if changed_routes is not None:
changed_routes[server] = route
elif route is None:
not_neighbours_anymore.append(server)
rc = RouteContainer(None, None, None)
server.set_route(rc)
if changed_routes is not None:
changed_routes[server] = rc
return not_neighbours_anymore
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app.config['SECURIZER'] = True
@self.app.route('/', methods=['GET', 'POST'])
@securizer
def home():
return {'msg': 'default response'}
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
db.init_app(self.app)
db.create_all()
self.server = Server('me', port=5000, me=True)
db.session.add(self.server)
self.dim = generate_dimension('dimension')
self.dim.current = True
db.session.add(self.dim)
self.token = create_access_token('test')
self.auth = HTTPBearerAuth(self.token)
self.url = 'https://me:5000/'
def setUp(self, mocked_now):
self.initials = dict(self.initials)
self.initials.update(action_template=False)
mocked_now.return_value = now1
super().setUp()
with self.app2_context:
mocked_now.return_value = now2
soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file')
at = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2', name='mkdir', version=1,
action_type=ActionType.SHELL, code='mkdir {dir}')
db.session.add_all([soft, at])
db.session.commit()
mocked_now.return_value = now3
ssa = SoftwareServerAssociation(software=soft, server=Server.get_current(), path='/root')
db.session.add(ssa)
db.session.commit()
self.soft_json = soft.to_json()
self.at_json = at.to_json()
self.catalog = fetch_catalog(now1)
self.mock_queue = mock.Mock()
self.mock_dm = mock.Mock()
self.mock_dm.flask_app = self.app
self.mock_dm.engine = db.engine
self.mock_dm.manager.dict.return_value = dict()
self.mock_dm.server_id = self.s1.id
self.cm = CatalogManager("Catalog", startup_event=threading.Event(), shutdown_event=threading.Event(),
publish_q=self.mock_queue, event_q=None, dimensigon=self.mock_dm)
db.session.commit()
def test_execute_send_software_error(self, mock_post, mock_get):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version=1, filename='test.zip')
node1 = Server('nodeA', port=5000)
ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/')
db.session.add_all([soft, node1, ssa1])
mock_post.return_value = Response(msg={'error': 'message'}, code=400)
mock_get.return_value = Response(code=400)
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(
dict(input=dict(software=str(soft.id), server=str(node1.id))),
timeout=10,
context=self.context)
mock_post.assert_called_once_with(node1,
'api_1_0.send',
json=dict(software_id=str(soft.id),
dest_server_id=str(
node1.id),
background=False,
include_transfer_data=True,
force=True),
timeout=10,
identity=ROOT)
self.assertFalse(cp.success)
self.assertEqual(flask.json.dumps(mock_post.return_value.msg),
cp.stdout)
def test_get_internal_error_server(self, m):
msg = '<html>Iternal error server</html>'
status = 500
responses.add(responses.GET, self.url, status=status, body=msg)
m.get(self.url, status=status, body=msg)
resp = get(Server.get_current(), 'home')
self.assertEqual(status, resp.code)
self.assertEqual(msg, resp.msg)
resp = run(async_get(Server.get_current(), 'home'))
self.assertEqual(status, resp.code)
self.assertEqual(msg, resp.msg)
def _proxy_request(request: 'flask.Request', destination: Server, verify=False) -> requests.Response:
url = destination.url() + request.full_path
req_data = request.get_json()
if request.path == '/ping':
server_data = {'id': str(g.server.id), 'name': g.server.name,
'time': get_now().strftime(defaults.DATETIME_FORMAT)}
if req_data:
if 'servers' not in req_data:
req_data['servers'] = {}
req_data['servers'].update({len(req_data['servers']) + 1: server_data})
else:
req_data = dict(servers={1: server_data})
kwargs = {
'json': req_data,
'allow_redirects': False
}
headers = {key.lower(): value for key, value in request.headers.items()}
# Let requests reset the host for us.
if 'host' in headers:
del headers['host']
headers['d-source'] = headers.get('d-source', '') + ':' + str(g.server.id)
kwargs['headers'] = headers
cookies = request.cookies
kwargs['cookies'] = cookies
return requests.request(request.method, url, verify=verify, **kwargs)
def cluster_in(server_id):
user = User.get_current()
data = request.get_json()
if user and user.name == 'root':
try:
keepalive = dt.datetime.strptime(data.get('keepalive'),
defaults.DATEMARK_FORMAT)
except ValueError:
raise errors.InvalidDateFormat(data.get('keepalive'),
defaults.DATEMARK_FORMAT)
current_app.dm.cluster_manager.put(server_id, keepalive)
_cluster_logger.debug(
f"{getattr(Server.query.get(server_id), 'name', server_id) or server_id} is a new alive server"
)
current_app.dm.route_manager.new_node_in_cluster(
server_id, data['routes'])
return {
'cluster':
current_app.dm.cluster_manager.get_cluster(
defaults.DATEMARK_FORMAT),
'neighbours': [s.id for s in Server.get_neighbours()]
}, 200
else:
raise errors.UserForbiddenError
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.headers = {
"Authorization":
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
db.create_all()
# set_initial(server=False)
self.srv1 = Server('node1',
id='00000000-0000-0000-0000-000000000001',
me=True)
self.file = File(source_server=self.srv1,
target='/etc/ssh/sshd_config',
id='00000000-0000-0000-0000-000000000002')
self.fsa = FileServerAssociation(file=self.file,
destination_server=self.srv1)
db.session.add_all([self.srv1, self.file, self.fsa])
db.session.commit()
def load_global_data_into_context():
from dimensigon.domain.entities import Server, Dimension
from dimensigon.web.decorators import set_source
global _dimension, _server
set_source()
g.server = Server.get_current()
g.dimension = Dimension.get_current()
def set_servers_and_routes(self):
self.n1 = Server(name='n1', dns_or_ip='1.1.1.1')
self.n2 = Server(name='n2', dns_or_ip='n2_dns')
self.n3 = Server(name='n3', port=8000)
self.r1 = Server(name='r1', dns_or_ip='3.3.3.3')
self.r2 = Server(name='r2', dns_or_ip='r2_dns')
db.session.add_all([self.n1, self.n2, self.r1, self.r2])
db.session.commit()
Route(destination=self.n1, cost=0)
Route(destination=self.n2, cost=0)
Route(destination=self.n3, cost=0)
Route(destination=self.r1, proxy_server_or_gate=self.n1, cost=1)
Route(destination=self.r2, proxy_server_or_gate=self.n2, cost=1)
db.session.commit()
