def fill_database(self, node):
db.create_all()
d = Dimension.from_json(self.dim)
d.current = True
s1 = Server(id='00000000-0000-0000-0000-000000000001', name='node1', created_on=now, me=node == 'node1')
g11 = Gate(id='00000000-0000-0000-0000-000000000011', server=s1, port=5000, dns=s1.name)
s2 = Server(id='00000000-0000-0000-0000-000000000002', name='node2', created_on=now, me=node == 'node2')
g12 = Gate(id='00000000-0000-0000-0000-000000000012', server=s2, port=5000, dns=s2.name)
s3 = Server(id='00000000-0000-0000-0000-000000000003', name='node3', created_on=now, me=node == 'node3')
g13 = Gate(id='00000000-0000-0000-0000-000000000013', server=s3, port=5000, dns=s3.name)
s4 = Server(id='00000000-0000-0000-0000-000000000004', name='node4', created_on=now, me=node == 'node4')
g14 = Gate(id='00000000-0000-0000-0000-000000000014', server=s4, port=5000, dns=s4.name)
if node == 'node1':
self.s1 = s1
self.s2 = s2
self.s3 = s3
self.s4 = s4
Route(s2, g12)
Route(s3, s2, 1)
elif node == 'node2':
Route(s1, g11)
Route(s3, g13)
elif node == 'node3':
Route(s1, s2, 1)
Route(s2, g12)
db.session.add_all([d, s1, s2, s3, s4])
db.session.commit()
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.headers = {
"Authorization":
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
db.create_all()
set_initial()
self.n1 = Server("node1", port=8000)
Route(self.n1, cost=0)
self.n2 = Server("node2", port=8000)
Route(self.n2, cost=0)
db.session.add_all([self.n1, self.n2])
db.session.commit()
self.datamark = Catalog.max_catalog(str)
def test_get_reachable_servers(self):
n1 = Server('n1', port=8000, id='22cd859d-ee91-4079-a112-000000000001')
n2 = Server('n2', port=8000, id='22cd859d-ee91-4079-a112-000000000002')
n3 = Server('n3', port=8000, id='22cd859d-ee91-4079-a112-000000000003')
n3.route = None
n4 = Server('n4', port=8000, id='22cd859d-ee91-4079-a112-000000000004')
r1 = Server('r1', port=8000, id='22cd859d-ee91-4079-a112-000000000011')
Route(destination=n1, cost=0)
Route(destination=n2, proxy_server_or_gate=n2.gates[0])
Route(destination=n4)
Route(destination=r1, proxy_server_or_gate=n1, cost=1)
me = Server('me', port=8000, me=True)
db.session.add_all([n1, n2, n3, n4, r1, me])
self.assertListEqual([n1, n2, r1], me.get_reachable_servers())
self.assertListEqual([n2, r1], me.get_reachable_servers(exclude=n1))
self.assertListEqual([n2, r1], me.get_reachable_servers(exclude=n1.id))
self.assertListEqual([r1],
me.get_reachable_servers(exclude=[n1.id, n2]))
def _fill_database(self, app: Flask):
with mock.patch('dimensigon.domain.entities.get_now') as mock_get_now:
mock_get_now.return_value = defaults.INITIAL_DATEMARK
node = app.config['SERVER_NAME']
with app.app_context():
db.create_all()
event.listen(db.session, 'after_commit', receive_after_commit)
set_initial(**self.initials)
d = Dimension.from_json(self.dim)
d.current = True
s1 = Server('node1',
created_on=defaults.INITIAL_DATEMARK,
id=self.SERVER1,
me=node == 'node1')
g11 = Gate(id='00000000-0000-0000-0000-000000000011',
server=s1,
port=5000,
dns=s1.name)
s2 = Server('node2',
created_on=defaults.INITIAL_DATEMARK,
id=self.SERVER2,
me=node == 'node2')
g12 = Gate(id='00000000-0000-0000-0000-000000000012',
server=s2,
port=5000,
dns=s2.name)
if node == 'node1':
Route(s2, g12)
elif node == 'node2':
Route(s1, g11)
self.fill_database()
db.session.add_all([d, s1, s2])
db.session.commit()
if node == 'node1':
self.s1, self.s2 = db.session.merge(s1), db.session.merge(s2)
def test_get_servers_from_scope_more_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
Server.set_initial()
servers = []
for i in range(0, 7):
s = Server(f'node{i}', port=5000, created_on=old_age)
if i == 0:
r = Route(s, cost=0)
else:
r = Route(s, random.choice(servers), cost=i)
db.session.add_all([s, r])
servers.append(s)
mock_get_now.return_value = dt.datetime(2019,
4,
2,
tzinfo=dt.timezone.utc)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
s62 = Server(f'node72', port=5000)
Route(s62, random.choice(servers), cost=6)
db.session.add(s62)
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(8, len(quorum))
self.assertNotIn(s62, quorum)
self.assertIn(s, quorum)
self.assertIn(Server.get_current(), quorum)
def test_upgrade_catalog_catalog_mismatch(self, mock_lock, mock_entities):
mock_lock.return_value.__enter__.return_value = 'applicant'
mock_entities.return_value = [('ActionTemplate', ActionTemplate),
('Server', Server)]
s = Server('server',
last_modified_at=dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc),
port=8000)
Route(s, cost=0)
at1 = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa1',
name='mkdir',
version=1,
action_type=ActionType.SHELL,
code='mkdir {dir}',
expected_output=None,
expected_rc=None,
system_kwargs={},
last_modified_at=dt.datetime(
2019, 4, 1, tzinfo=dt.timezone.utc))
at2 = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2',
name='rmdir',
version=1,
action_type=ActionType.SHELL,
code='rmdir {dir}',
expected_output=None,
expected_rc=None,
system_kwargs={},
last_modified_at=dt.datetime(
2019, 4, 2, tzinfo=dt.timezone.utc))
responses.add(
method='GET',
url=re.compile('^' + s.url(
'api_1_0.catalog', data_mark='12345').replace('12345', '')),
json={"ActionTemplate": [at1.to_json(),
at2.to_json()]})
with self.assertRaises(errors.CatalogMismatch):
upgrade_catalog_from_server(s)
def test_sqlalchemy_entity_events_server(self, mock_now):
mock_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc)
s = Server('source', port=5000)
d = Server('destination', port=5000)
db.session.add_all([s, d])
db.session.commit()
mock_now.return_value = dt.datetime(2019, 4, 2, tzinfo=dt.timezone.utc)
s.route = Route(destination=d, proxy_server_or_gate=d.gates[0], cost=0)
db.session.commit()
self.assertEqual(dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc),
s.last_modified_at)
g = s.add_new_gate(dns_or_ip='dns', port=5000)
db.session.add(g)
db.session.commit()
self.assertEqual(dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc),
s.last_modified_at)
def setUp(self):
"""Create and configure a new self.app instance for each test."""
# create a temporary file to isolate the database for each test
# create the self.app with common test config
self.app = Flask(__name__)
@self.app.route('/', methods=['GET', 'POST'])
@securizer
def hello():
return request.get_json() or {'msg': 'default response'}
@self.app.route('/join', methods=['POST'])
@securizer
def join():
return {'msg': 'default response'}
@self.app.route('/empty', methods=['GET'])
@securizer
def empty():
return "", 204
@self.app.route('/list', methods=['GET'])
@securizer
def list():
return [1, 2], 200
self.app.config['SECURIZER'] = True
self.app.config['SECURIZER_PLAIN'] = True
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
db.init_app(self.app)
db.create_all()
self.d = generate_dimension('test')
self.d.current = True
self.srv1 = Server(id='bbbbbbbb-1234-5678-1234-56781234bbb1', name='server1',
dns_or_ip='192.168.1.9', port=7123, me=True)
Route(self.srv1, cost=0)
db.session.add_all([self.srv1, self.d])
db.session.commit()
def test_upgrade_catalog_no_data(self, mock_lock, mock_entities):
mock_lock.return_value.__enter__.return_value = 'applicant'
mock_entities.return_value = [('ActionTemplate', ActionTemplate)]
s = Server('server',
last_modified_at=dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc))
g = Gate(server=s, port=80)
Route(s, cost=0)
responses.add(
method='GET',
url=re.compile('^' + s.url(
'api_1_0.catalog', data_mark='12345').replace('12345', '')),
json={"ActionTemplate": []})
upgrade_catalog_from_server(s)
atl = [at.to_json() for at in ActionTemplate.query.all()]
self.assertEqual(0, len(atl))
def _update_route_table_from_data(
self,
new_routes: t.Dict,
auth=None) -> t.Dict[Server, RouteContainer]:
changed_routes = {}
routes = new_routes.get('route_list', [])
routes.sort(key=lambda x: x.get('cost') or MAX_COST, reverse=True)
try:
likely_proxy_server = self.session.query(Server).get(
new_routes.get('server_id'))
if not likely_proxy_server:
self.logger.warning(
f"Server id still '{new_routes.get('server_id')}' not created."
)
return changed_routes
debug_new_routes = []
for new_route in routes:
target_server = self.session.query(Server).get(
new_route.get('destination_id'))
proxy_server = self.session.query(Server).get(
new_route.get('proxy_server_id'))
gate = self.session.query(Gate).get(new_route.get('gate_id'))
dest_name = getattr(target_server, 'name',
new_route.get('destination_id'))
proxy_name = getattr(proxy_server, 'name',
new_route.get('proxy_server_id'))
gate_str = str(gate) if gate else new_route.get('gate_id')
cost = new_route.get('cost')
if gate_str and proxy_name:
gate_str = gate_str + '*' + proxy_name
debug_new_routes.append(
f'{dest_name} -> {gate_str or proxy_name} / {cost}')
if target_server is None:
self.logger.warning(
f"Destination server unknown {new_route.get('destination_id')}"
)
continue
if target_server.id == self.server.id:
# check if server has detected me as a neighbour
if new_route.get('cost') == 0:
# check if I do not have it as a neighbour yet
if likely_proxy_server.route and likely_proxy_server.route.cost != 0:
# check if I have a gate to contact with it
route = check_gates(likely_proxy_server)
if isinstance(route, RouteContainer):
changed_routes[likely_proxy_server] = route
likely_proxy_server.set_route(route)
else:
# server may be created without route (backward compatibility)
if target_server.route is None:
target_server.set_route(
Route(destination=target_server))
# process routes whose proxy_server is not me
if self.server.id != new_route.get('proxy_server_id'):
# check If I reach destination
if target_server.route.cost is not None:
if new_route.get('cost') is None:
# likely proxy does not reach but I reach it. It might be shutdown unexpectedly?
if target_server.route.cost == 0:
# check if I still have it as a neighbour
route = check_gates(target_server)
else:
if target_server.route.proxy_server == likely_proxy_server:
route = RouteContainer(
None, None, None)
else:
# check if I still have access through my proxy
cost, time = ntwrk.ping(
target_server,
retries=1,
timeout=20,
session=self.session)
if cost == target_server.route.cost:
# still a valid route
route = target_server.route
elif cost is None:
route = RouteContainer(
None, None, None)
else:
route = RouteContainer(
target_server.route.
proxy_server, None, cost)
if isinstance(route, RouteContainer):
# gate changed
changed_routes[target_server] = route
target_server.set_route(route)
elif route is None:
# no route to host. I've lost contact too
changed_routes[
target_server] = RouteContainer(
None, None, None)
target_server.set_route(
changed_routes[target_server])
else:
# still a valid route. Send route to likely_proxy_server to tell it I have access
resp = ntwrk.patch(
likely_proxy_server,
'api_1_0.routes',
json=dict(
server_id=str(self.server.id),
route_list=[
target_server.route.to_json()
]),
auth=get_root_auth(),
timeout=5)
if not resp.ok:
self.logger.info(
f'Unable to send route to {likely_proxy_server}: {resp}'
)
elif target_server.route.proxy_server is not None and \
target_server.route.proxy_server == likely_proxy_server:
# my proxy is telling me the route has changed
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
target_server.set_route(rc)
changed_routes.update({target_server: rc})
elif new_route.get(
'cost') + 1 < target_server.route.cost:
# if new route has less cost than actual route, take it as my new route
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
target_server.set_route(rc)
changed_routes.update({target_server: rc})
else:
# me does not reaches target_server
# if new route reaches the destination take it as a new one
if (new_route.get('cost') == 0
and new_route.get('gate_id') is not None
) or (new_route.get('cost') is not None
and proxy_server is not None):
rc = RouteContainer(likely_proxy_server, None,
new_route.get('cost') + 1)
else:
# neither my route and the new route has access to the destination
rc = RouteContainer(None, None, None)
if target_server.route and (
rc.proxy_server !=
target_server.route.proxy_server
or rc.gate != target_server.route.gate
or rc.cost != target_server.route.cost):
target_server.set_route(rc)
changed_routes.update({target_server: rc})
else:
# target_server reached through me as a proxy from likely_proxy
pass
query = self.session.query(Route).filter(
Route.proxy_server_id.in_([
s.id for s, r in changed_routes.items() if r.cost is None
]))
rc = RouteContainer(None, None, None)
for route in query.all():
route.set_route(rc)
changed_routes[route.destination] = rc
self.logger.debug(
f"New routes processed from {likely_proxy_server.name}: {json.dumps(debug_new_routes, indent=2)}"
)
# if changed_routes:
# Parameter.set('routing_last_refresh', get_now())
except errors.InvalidRoute as e:
debug_new_routes = []
routes.sort(key=lambda x: x.get('cost') or MAX_COST, reverse=True)
for new_route in routes:
target_server = self.session.query(Server).get(
new_route.get('destination_id'))
proxy_server = self.session.query(Server).get(
new_route.get('proxy_server_id'))
gate = self.session.query(Gate).get(new_route.get('gate_id'))
dest_name = getattr(target_server, 'name',
new_route.get('destination_id'))
proxy_name = getattr(proxy_server, 'name',
new_route.get('proxy_server_id'))
gate_str = str(gate) if gate else new_route.get('gate_id')
cost = new_route.get('cost')
if gate_str and proxy_name:
gate_str = gate_str + '*' + proxy_name
debug_new_routes.append(
f'{dest_name} -> {gate_str or proxy_name} / {cost}')
self.logger.exception(
"Error setting routes from following data: " +
json.dumps(debug_new_routes, indent=4))
return changed_routes
def _route_table_merge(self, data: t.Dict[Server, ntwrk.Response]):
changed_routes: t.Dict[Server, RouteContainer] = {}
temp_table_routes: t.Dict[uuid.UUID, t.List[RouteContainer]] = {}
for s, resp in data.items():
if resp.code == 200:
server_id = resp.msg.get(
'server_id', None) or resp.msg.get('server').get('id')
likely_proxy_server_entity = self.session.query(Server).get(
server_id)
for route_json in resp.msg['route_list']:
route_json = convert(route_json)
if route_json.destination_id != self.server.id \
and route_json.proxy_server_id != self.server.id \
and route_json.gate_id not in [g.id for g in self.server.gates]:
if route_json.destination_id not in temp_table_routes:
temp_table_routes.update(
{route_json.destination_id: []})
if route_json.cost is not None:
route_json.cost += 1
route_json.proxy_server_id = likely_proxy_server_entity.id
route_json.gate_id = None
temp_table_routes[
route_json.destination_id].append(
RouteContainer(
likely_proxy_server_entity.id, None,
route_json.cost))
elif route_json.cost is None:
# remove a routing if gateway cannot reach the destination
temp_table_routes[
route_json.destination_id].append(
RouteContainer(route_json.proxy_server_id,
None, None))
else:
self.logger.error(
f"Error while connecting with {s}. Error: {resp}")
# Select new routes based on neighbour routes
neighbour_ids = [
s.id for s in Server.get_neighbours(session=self.session)
]
for destination_id in filter(lambda s: s not in neighbour_ids,
temp_table_routes.keys()):
route = self.session.query(Route).filter_by(
destination_id=destination_id).one_or_none()
if not route:
server = self.session.query(Server).get(destination_id)
if not server:
continue
else:
route = Route(destination=server)
temp_table_routes[destination_id].sort(
key=lambda x: x.cost or MAX_COST)
if len(temp_table_routes[destination_id]) > 0:
min_route = temp_table_routes[destination_id][0]
proxy_server: Server = self.session.query(Server).get(
min_route.proxy_server)
cost = min_route.cost
if route.proxy_server != proxy_server or route.cost != cost:
rc = RouteContainer(proxy_server, None, cost)
route.set_route(rc)
changed_routes[route.destination] = rc
self.session.add(route)
data = {}
for server, temp_route in changed_routes.items():
data.update({
str(server): {
'proxy_server': str(temp_route.proxy_server),
'gate': str(temp_route.gate),
'cost': str(temp_route.cost)
}
})
return changed_routes
def test_upgrade_catalog(self, mock_lock, mock_entities, mock_now,
mock_tzlocal):
mock_lock.return_value.__enter__.return_value = 'applicant'
mock_entities.return_value = [('ActionTemplate', ActionTemplate),
('Server', Server), ('Gate', Gate)]
mock_now.return_value = dt.datetime(2019, 4, 1, tzinfo=dt.timezone.utc)
mock_tzlocal.return_value = dt.timezone.utc
at1 = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa1',
name='mkdir',
version=1,
action_type=ActionType.SHELL,
code='mkdir {dir}',
expected_output=None,
expected_rc=None,
system_kwargs={})
db.session.add(at1)
db.session.commit()
s = Server(id='aaaaaaaa-1234-5678-1234-56781234bbb1',
name='server',
last_modified_at=dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc))
s_json = s.to_json()
g = Gate(server=s,
port=80,
dns='server',
last_modified_at=dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc))
g_json = g.to_json()
Route(s, cost=0)
at2 = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2',
name='rmdir',
version=1,
action_type=ActionType.SHELL,
code='rmdir {dir}',
expected_output=None,
expected_rc=None,
system_kwargs={},
last_modified_at=dt.datetime(
2019, 4, 2, tzinfo=dt.timezone.utc))
at2_json = at2.to_json()
del at2
at1_json = at1.to_json()
del at1
at1_json['code'] = 'mkdir -p {dir}'
responses.add(
method='GET',
url=re.compile('^' + s.url(
'api_1_0.catalog', data_mark='12345').replace('12345', '')),
json={
"ActionTemplate": [at1_json, at2_json],
"Server": [s_json],
"Gate": [g_json]
})
upgrade_catalog_from_server(s)
db.session.expire_all()
atl = [at.to_json() for at in ActionTemplate.query.all()]
self.assertListEqual([at1_json, at2_json], atl)
c = Catalog.query.get('ActionTemplate')
self.assertEqual(dt.datetime(2019, 4, 2, tzinfo=dt.timezone.utc),
c.last_modified_at)
at1 = ActionTemplate.query.get('aaaaaaaa-1234-5678-1234-56781234aaa1')
self.assertEqual('mkdir -p {dir}', at.code)
def test_route(self):
dest = Server('dest', port=80)
proxy = Server('proxy', port=80)
r = Route(destination=dest)
self.assertEqual(dest, r.destination)
self.assertIsNone(r.proxy_server)
self.assertIsNone(r.gate)
self.assertIsNone(r.cost)
# routes defined with a gate
## dest
r = Route(destination=dest, proxy_server_or_gate=dest.gates[0])
self.assertEqual(dest, r.destination)
self.assertIsNone(r.proxy_server)
self.assertEqual(dest.gates[0], r.gate)
self.assertEqual(0, r.cost)
r = Route(destination=dest, proxy_server_or_gate=dest.gates[0], cost=0)
self.assertEqual(dest, r.destination)
self.assertIsNone(r.proxy_server)
self.assertEqual(dest.gates[0], r.gate)
self.assertEqual(0, r.cost)
with self.assertRaises(ValueError):
r = Route(destination=dest,
proxy_server_or_gate=dest.gates[0],
cost=1)
## proxy
with self.assertRaises(ValueError):
Route(destination=dest, proxy_server_or_gate=proxy.gates[0])
with self.assertRaises(ValueError):
Route(destination=dest,
proxy_server_or_gate=proxy.gates[0],
cost=0)
r = Route(destination=dest,
proxy_server_or_gate=proxy.gates[0],
cost=1)
self.assertEqual(dest, r.destination)
self.assertEqual(proxy, r.proxy_server)
self.assertIsNone(r.gate)
self.assertEqual(1, r.cost)
# routes defined with a proxy server
## dest
with self.assertRaises(ValueError):
r = Route(destination=dest, proxy_server_or_gate=dest)
with self.assertRaises(ValueError):
r = Route(destination=dest, proxy_server_or_gate=dest, cost=0)
with self.assertRaises(ValueError):
r = Route(destination=dest, proxy_server_or_gate=dest, cost=1)
## proxy
with self.assertRaises(ValueError):
r = Route(destination=dest, proxy_server_or_gate=proxy)
with self.assertRaises(ValueError):
r = Route(destination=dest, proxy_server_or_gate=proxy, cost=0)
r = Route(destination=dest, proxy_server_or_gate=proxy, cost=1)
self.assertEqual(dest, r.destination)
self.assertEqual(proxy, r.proxy_server)
self.assertIsNone(r.gate)
self.assertEqual(1, r.cost)
