def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.headers = {
"Authorization":
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
db.create_all()
set_initial()
self.soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1',
name='test',
version='1',
filename='file')
self.soft2 = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa2',
name='test',
version='2',
filename='file')
self.ssa = SoftwareServerAssociation(software=self.soft,
server=Server.get_current(),
path='/root')
db.session.add_all([self.soft, self.soft2, self.ssa])
db.session.commit()
def test_get_servers_from_scope_less_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
me = Server(id='00000000-0000-0000-0000-000000000000',
name='node0',
port=5000,
me=True,
created_on=now)
db.session.add(me)
servers = []
for i in range(1, 4):
s = Server(f'node{i}', port=5000, created_on=old_age)
r = Route(s, cost=0)
db.session.add_all([s, r])
servers.append(s)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(4, len(quorum))
self.assertIn(Server.get_current(), quorum)
def _notify_cluster_out(self):
with self.dm.flask_app.app_context():
servers = Server.get_neighbours()
if servers:
self.logger.debug(
f"Sending shutdown to {', '.join([s.name for s in servers])}"
)
else:
self.logger.debug("No server to send shutdown information")
if servers:
responses = asyncio.run(
ntwrk.parallel_requests(
servers,
'post',
view_or_url='api_1_0.cluster_out',
view_data=dict(server_id=str(Server.get_current().id)),
json={
'death':
get_now().strftime(defaults.DATEMARK_FORMAT)
},
timeout=2,
auth=get_root_auth()))
if self.logger.level <= logging.DEBUG:
for r in responses:
if not r.ok:
self.logger.warning(
f"Unable to send data to {r.server}: {r}")
def setUp(self, mocked_now):
self.initials = dict(self.initials)
self.initials.update(action_template=False)
mocked_now.return_value = now1
super().setUp()
with self.app2_context:
mocked_now.return_value = now2
soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file')
at = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2', name='mkdir', version=1,
action_type=ActionType.SHELL, code='mkdir {dir}')
db.session.add_all([soft, at])
db.session.commit()
mocked_now.return_value = now3
ssa = SoftwareServerAssociation(software=soft, server=Server.get_current(), path='/root')
db.session.add(ssa)
db.session.commit()
self.soft_json = soft.to_json()
self.at_json = at.to_json()
self.catalog = fetch_catalog(now1)
self.mock_queue = mock.Mock()
self.mock_dm = mock.Mock()
self.mock_dm.flask_app = self.app
self.mock_dm.engine = db.engine
self.mock_dm.manager.dict.return_value = dict()
self.mock_dm.server_id = self.s1.id
self.cm = CatalogManager("Catalog", startup_event=threading.Event(), shutdown_event=threading.Event(),
publish_q=self.mock_queue, event_q=None, dimensigon=self.mock_dm)
db.session.commit()
def test_get_internal_error_server(self, m):
msg = '<html>Iternal error server</html>'
status = 500
responses.add(responses.GET, self.url, status=status, body=msg)
m.get(self.url, status=status, body=msg)
resp = get(Server.get_current(), 'home')
self.assertEqual(status, resp.code)
self.assertEqual(msg, resp.msg)
resp = run(async_get(Server.get_current(), 'home'))
self.assertEqual(status, resp.code)
self.assertEqual(msg, resp.msg)
def load_global_data_into_context():
from dimensigon.domain.entities import Server, Dimension
from dimensigon.web.decorators import set_source
global _dimension, _server
set_source()
g.server = Server.get_current()
g.dimension = Dimension.get_current()
def test_from_json_new(self):
ssa_json = dict(software_id='aaaaaaaa-1234-5678-1234-56781234aaa2',
server_id=str(Server.get_current().id),
path='/root',
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
smashed = SoftwareServerAssociation.from_json(ssa_json)
self.assertEqual(defaults.INITIAL_DATEMARK, smashed.last_modified_at)
self.assertIsNotNone(smashed.last_modified_at)
self.assertEqual("/root", smashed.path)
self.assertIsNotNone(smashed.path)
self.assertEqual(Server.get_current(), smashed.server)
self.assertIsNotNone(smashed.server)
self.assertEqual(self.soft2, smashed.software)
self.assertIsNotNone(smashed.software)
def callback_prevent(url, **kwargs):
assert kwargs['json'] == {
'scope': 'CATALOG',
'datemark': self.datemark,
'applicant': [Server.get_current().id, self.n1.id, self.n2.id]
}
return CallbackResult("{'message': 'Preventing lock acquired'}",
status=200)
def test_lock_catalog_error_on_preventing(self, m):
def callback_prevent(url, **kwargs):
self.assertDictEqual(
kwargs['json'], {
'scope':
'CATALOG',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
})
return CallbackResult("{'message': 'Preventing lock acquired'}",
status=200)
def callback_unlock(url, **kwargs):
self.assertDictEqual(
kwargs['json'], {
'scope':
'CATALOG',
'action':
'UNLOCK',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
})
return CallbackResult("{'message': 'UnLocked'}", status=200)
m.post(Server.get_current().url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(self.n1.url('api_1_0.locker_prevent'),
exception=ClientConnectionError())
m.post(self.n2.url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(Server.get_current().url('api_1_0.locker_unlock'),
callback=callback_unlock)
m.post(self.n2.url('api_1_0.locker_unlock'), callback=callback_unlock)
with self.assertRaises(errors.LockError):
lock(Scope.CATALOG, [Server.get_current(), self.n1, self.n2],
identity=ROOT)
c = Locker.query.get(Scope.CATALOG)
self.assertEqual(State.UNLOCKED, c.state)
def __init__(self, server: 'Server', proxy: 'Server' = None):
from dimensigon.domain.entities import Server
try:
p = proxy or Server.get_current()
except NoResultFound:
p = None
self.destination = dict(name=server.name, id=server.id)
self.proxy = dict(name=p.name, id=p.id) if p else None
def callback_lock(url, **kwargs):
assert kwargs['json'] == {
'scope':
'CATALOG',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
}
return CallbackResult("{'message': 'Locked'}", status=200)
def ping(dest: t.Union[Server, Gate],
retries=3,
timeout=30,
verify=False,
session=None):
server = Server.get_current(session=session)
return _ping(dest=dest,
source=server,
retries=retries,
timeout=timeout,
verify=verify)
def make_first_request(self):
from dimensigon.domain.entities import Server
import dimensigon.web.network as ntwrk
with self.flask_app.app_context():
start = time.time()
while True:
resp = ntwrk.get(Server.get_current(), 'root.home', timeout=1)
if not resp.ok and time.time() - start < 30:
time.sleep(0.5)
else:
break
self._main_ctx.publish_q.safe_put(EventMessage("Listening", source="Dimensigon"))
def callback_prevent(url, **kwargs):
self.assertDictEqual(
kwargs['json'], {
'scope':
'CATALOG',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
})
return CallbackResult("{'message': 'Preventing lock acquired'}",
status=200)
def test_lock_catalog(self, m):
def callback_prevent(url, **kwargs):
assert kwargs['json'] == {
'scope': 'CATALOG',
'datemark': self.datemark,
'applicant': [Server.get_current().id, self.n1.id, self.n2.id]
}
return CallbackResult("{'message': 'Preventing lock acquired'}",
status=200)
def callback_lock(url, **kwargs):
assert kwargs['json'] == {
'scope':
'CATALOG',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
}
return CallbackResult("{'message': 'Locked'}", status=200)
m.post(Server.get_current().url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(self.n1.url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(self.n2.url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(Server.get_current().url('api_1_0.locker_lock'),
callback=callback_lock)
m.post(self.n1.url('api_1_0.locker_lock'), callback=callback_lock)
m.post(self.n2.url('api_1_0.locker_lock'), callback=callback_lock)
applicant = lock(Scope.CATALOG,
[Server.get_current(), self.n1, self.n2],
identity=ROOT)
self.assertEqual(applicant,
[Server.get_current().id, self.n1.id, self.n2.id])
def test_to_from_json(self):
ssa_json = self.ssa.to_json()
ssa_json['path'] = '/new_root'
smashed = SoftwareServerAssociation.from_json(ssa_json)
self.assertEqual("/new_root", smashed.path)
self.assertIsNotNone(smashed.path)
self.assertEqual(Server.get_current(), smashed.server)
self.assertIsNotNone(smashed.server)
self.assertEqual(self.soft, smashed.software)
self.assertIsNotNone(smashed.software)
self.assertEqual(self.ssa.last_modified_at, smashed.last_modified_at)
self.assertIsNotNone(smashed.last_modified_at)
db.session.commit()
del smashed
ssa = SoftwareServerAssociation.query.get(
('aaaaaaaa-1234-5678-1234-56781234aaa1', Server.get_current().id))
self.assertEqual("/new_root", ssa.path)
def callback_unlock(url, **kwargs):
self.assertDictEqual(
kwargs['json'], {
'scope':
'CATALOG',
'action':
'UNLOCK',
'applicant': [
str(Server.get_current().id),
str(self.n1.id),
str(self.n2.id)
]
})
return CallbackResult("{'message': 'UnLocked'}", status=200)
def test_get(self):
resp = self.client.get(url_for('api_1_0.loglist'),
headers=self.auth.header)
self.assertListEqual([self.log.to_json()], resp.get_json())
log = Log(source_server=Server.get_current(),
target='/access.log',
destination_server=self.dest2)
db.session.add(log)
db.session.commit()
# test with filter
resp = self.client.get(url_for('api_1_0.loglist') +
"?filter[target]=/access.log",
headers=self.auth.header)
self.assertListEqual([log.to_json()], resp.get_json())
# test with filter on a server
resp = self.client.get(url_for('api_1_0.loglist') +
f"?filter[dst_server_id]={self.dest1.id}",
headers=self.auth.header)
self.assertListEqual([self.log.to_json()], resp.get_json())
def test_get_servers_from_scope_more_than_min_quorum(
self, mock_get_now, mock_app):
mock_get_now.return_value = dt.datetime(2019,
4,
1,
tzinfo=dt.timezone.utc)
Server.set_initial()
servers = []
for i in range(0, 7):
s = Server(f'node{i}', port=5000, created_on=old_age)
if i == 0:
r = Route(s, cost=0)
else:
r = Route(s, random.choice(servers), cost=i)
db.session.add_all([s, r])
servers.append(s)
mock_get_now.return_value = dt.datetime(2019,
4,
2,
tzinfo=dt.timezone.utc)
mock_app.dm.cluster_manager.get_alive.return_value = [
s.id for s in servers
]
s62 = Server(f'node72', port=5000)
Route(s62, random.choice(servers), cost=6)
db.session.add(s62)
quorum = get_servers_from_scope(scope=Scope.CATALOG)
self.assertEqual(8, len(quorum))
self.assertNotIn(s62, quorum)
self.assertIn(s, quorum)
self.assertIn(Server.get_current(), quorum)
def pack_msg(data, *args, **kwargs):
if not __ca.config['SECURIZER']:
return data
else:
# if not ('symmetric_key' in kwargs or 'cipher_key' in kwargs):
# try:
# kwargs['cipher_key'] = session.get('cipher_key')
# except RuntimeError:
# pass
# if generate_key:
# kwargs.pop('symmetric_key', None)
# kwargs.pop('cipher_key', None)
dim = None
if dim is None:
dim = Dimension.get_current()
if dim is None:
raise ValueError('No dimension found but SECURIZER set')
return _pack_msg(data,
*args,
source=Server.get_current(),
pub_key=dim.public,
priv_key=dim.private,
**kwargs)
def _prepare_headers(server: t.Union[Server, str], headers=None):
headers = headers or {}
if isinstance(server, Server):
headers.update({'D-Destination': str(server.id)})
headers.update({'D-Source': str(Server.get_current().id)})
return headers
def test_lock_scope_packing(self, mock_app, m):
mock_app.dm.cluster_manager.get_alive.return_value = [
self.s2.id, self.s3.id
]
def callback_prevent(url, **kwargs):
return CallbackResult("{'message': 'Preventing lock acquired'}",
status=200)
def callback_lock(url, **kwargs):
return CallbackResult("{'message': 'Locked'}", status=200)
def callback_unlock(url, **kwargs):
return CallbackResult("{'message': 'UnLocked'}", status=200)
def callback_client(url, **kwargs):
kwargs.pop('allow_redirects')
# workarround for https://github.com/pnuckowski/aioresponses/issues/111
headers = {
'Authorization':
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
r = self.client.post(url.path,
json=kwargs['json'],
headers=headers)
return CallbackResult(r.data, status=r.status_code)
m.post(re.compile(Server.get_current().url() + '.*'),
callback=callback_client)
m.post(self.s2.url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(self.s3.url('api_1_0.locker_prevent'),
callback=callback_prevent)
m.post(re.compile(Server.get_current().url() + '.*'),
callback=callback_client)
m.post(self.s2.url('api_1_0.locker_lock'), callback=callback_lock)
m.post(self.s3.url('api_1_0.locker_lock'), callback=callback_lock)
m.post(re.compile(Server.get_current().url() + '.*'),
callback=callback_client)
m.post(self.s2.url('api_1_0.locker_unlock'), callback=callback_unlock)
m.post(self.s3.url('api_1_0.locker_unlock'), callback=callback_unlock)
l = Locker.query.get(Scope.CATALOG)
self.assertEqual(State.UNLOCKED, l.state)
self.assertEqual(None, l.applicant)
with self.app.test_request_context('/api/v1.0/lock'):
load_global_data_into_context()
with lock_scope(Scope.CATALOG, identity=ROOT):
l = Locker.query.get(Scope.CATALOG)
self.assertEqual(State.LOCKED, l.state)
self.assertEqual([
str(Server.get_current().id),
str(self.s2.id),
str(self.s3.id)
], l.applicant)
l = Locker.query.get(Scope.CATALOG)
self.assertEqual(State.UNLOCKED, l.state)
self.assertEqual(None, l.applicant)
def bootstrap(self):
""" bootstraps the application. Gunicorn is still not listening on sockets
"""
with self.app_context():
from dimensigon.domain.entities import Server, Parameter
import dimensigon.web.network as ntwrk
from dimensigon.domain.entities import Locker
# reset scopes
Locker.set_initial(unlock=True)
# check gates
me = Server.get_current()
if me is None:
raise RuntimeError("No server set as 'current'")
input_gates = bind2gate(self.dm.config.http_conf.get('bind'))
current_gates = [(gate.dns or str(gate.ip), gate.port)
for gate in me.gates]
new_gates = set(input_gates).difference(set(current_gates))
self.server_id_with_new_gates = None
if new_gates:
if Parameter.get('join_server'):
join_server = Server.query.get(
Parameter.get('join_server'))
else:
join_server = None
servers = Server.get_neighbours()
if join_server in servers:
servers.pop(servers.index(join_server))
servers.append(join_server)
else:
self.logger.warning(
f'Join server {join_server} is not a neighbour')
start = time.time()
resp = None
server = True
while len(servers) > 0 and server and (time.time() -
start) < 900:
server_retries = 0
server = servers[-1]
self.logger.debug(
f"Sending new gates {new_gates} to {server}...")
resp = ntwrk.patch(
server,
'api_1_0.serverresource',
view_data=dict(server_id=str(Server.get_current().id)),
json={
'gates': [{
'dns_or_ip': ip,
'port': port
} for ip, port in new_gates]
},
timeout=60,
auth=get_root_auth())
if not resp.ok:
self.logger.debug(
f"Unable to send new gates to {server}. Reason: {resp}"
)
self.logger.info(
f"Unable to create new gates. Trying to send again in 5 seconds..."
)
time.sleep(5)
if resp.code == 409:
# try with the same server
server_retries += 1
elif resp.code == 500:
# try with another server
i = servers.index(server) - 1
if i >= 0:
server = servers[i]
server_retries = 0
else:
server = None
if server_retries == 3:
# changing server
i = servers.index(server) - 1
if i >= 0:
server = servers[i]
server_retries = 0
else:
server = None
else:
self.logger.debug("New gates created succesfully")
Parameter.set('new_gates_server', server.id)
break
if not servers:
if Server.query.count() == 1:
self.logger.info(
f"Creating new gates {new_gates} without performing a lock on catalog"
)
for gate in new_gates:
g = me.add_new_gate(gate[0], gate[1])
db.session.add(g)
else:
if resp and not resp.ok:
self.logger.warning(
f"Remote servers may not connect with {me}. ")
db.session.commit()
def deploy_orchestration(orchestration: t.Union[Id, Orchestration],
hosts: t.Dict[str, t.Union[t.List[Id]]],
var_context: 'Context' = None,
execution: t.Union[Id, OrchExecution] = None,
executor: t.Union[Id, User] = None,
execution_server: t.Union[Id, Server] = None,
lock_retries=2,
lock_delay=3, timeout=None) -> Id:
"""deploy the orchestration
Args:
orchestration: id or orchestration to execute
hosts: Mapping to all distributions
var_context: Context configuration
execution: id or execution to associate with the orchestration. If none, a new one is created
executor: id or User who executes the orchestration
execution_server: id or User who executes the orchestration
lock_retries: tries to lock for orchestration N times
lock_delay: delay between retries
Returns:
OrchExecution ID
Raises:
Exception: if anything goes wrong
"""
execution = execution or var_context.env.get('orch_execution_id')
executor = executor or var_context.env.get('executor_id')
hosts = hosts or var_context.get('hosts')
if not isinstance(orchestration, Orchestration):
orchestration = db.session.query(Orchestration).get(orchestration)
if not isinstance(execution, OrchExecution):
exe = None
if execution is not None:
exe = db.session.query(OrchExecution).get(execution)
if exe is None:
if not isinstance(executor, User):
executor = db.session.query(User).get(executor)
if executor is None:
raise ValueError('executor must be set')
if not isinstance(execution_server, Server):
if execution_server is None:
try:
execution_server = g.server
except AttributeError:
execution_server = Server.get_current()
if execution_server is None:
raise ValueError('execution server not found')
else:
execution_server = db.session.query(Server).get(execution_server)
exe = OrchExecution(id=execution, orchestration_id=orchestration.id, target=hosts,
params=dict(var_context),
executor_id=executor.id, server_id=execution_server.id)
db.session.add(exe)
db.session.commit()
else:
exe = execution
current_app.logger.debug(
f"Execution {exe.id}: Launching orchestration {orchestration} on {hosts} with {var_context}")
return _deploy_orchestration(orchestration, var_context, hosts, exe, lock_retries, lock_delay, timeout)
def my_logs(self):
return self.session.query(Log).filter_by(
source_server=Server.get_current(session=self.session)).all()
def launch_orchestration(orchestration_id):
data = request.get_json()
if orchestration_id:
orchestration = Orchestration.query.get_or_raise(orchestration_id)
else:
iden = (data.get('orchestration'), )
columns = ('orchestration', )
query = Orchestration.query.filter_by(name=data.get('orchestration'))
if 'version' in data:
iden += (data.get('version'), )
columns += ('version', )
query = query.filter_by(version=data.get('version'))
query = query.order_by(Orchestration.version.desc())
if query.count() <= 1:
orchestration = query.one_or_none()
else:
orchestration = query.first()
if not orchestration:
raise errors.EntityNotFound('Orchestration', iden, columns)
if not orchestration.steps:
return errors.GenericError(
'orchestration does not have steps to execute',
orchestration_id=orchestration_id)
params = data.get('params') or {}
hosts = data.get('hosts', Server.get_current().id)
a = set(orchestration.target)
if not isinstance(hosts, dict):
hosts = dict(all=hosts)
b = set(hosts.keys())
c = a - b
if len(c) > 0:
raise errors.TargetUnspecified(c)
c = b - a
if len(c) > 0:
raise errors.TargetNotNeeded(c)
not_found = normalize_hosts(hosts)
if not_found:
raise errors.ServerNormalizationError(not_found)
for target, target_hosts in hosts.items():
if len(target_hosts) == 0:
raise errors.EmptyTarget(target)
# check param entries
# rest = orchestration.user_parameters - set(params.keys())
# if rest:
# rest = list(rest)
# rest.sort()
# return {'error': f"Parameter(s) not specified: {', '.join(rest)}"}, 404
execution_id = str(uuid.uuid4())
executor_id = get_jwt_identity()
vc = Context(params,
dict(execution_id=None,
root_orch_execution_id=execution_id,
orch_execution_id=execution_id,
executor_id=executor_id),
vault=Vault.get_variables_from(executor_id,
scope=data.get(
'scope', 'global')))
if not data.get('skip_validation', False):
validate_input_chain(
orchestration, {
'input': set(params.keys()),
'env': set(vc.env.keys()),
'vault': set(vc.vault.keys())
})
if request.get_json().get('background', True):
future = executor.submit(deploy_orchestration,
orchestration=orchestration.id,
var_context=vc,
hosts=hosts,
timeout=data.get('timeout', None))
try:
future.result(5)
except concurrent.futures.TimeoutError:
return {'execution_id': execution_id}, 202
except Exception as e:
current_app.logger.exception(
f"Exception got when executing orchestration {orchestration}")
raise
else:
try:
deploy_orchestration(orchestration=orchestration,
var_context=vc,
hosts=hosts,
timeout=data.get('timeout', None))
except Exception as e:
current_app.logger.exception(
f"Exception got when executing orchestration {orchestration}")
raise
return OrchExecution.query.get(execution_id).to_json(
add_step_exec=True,
human=check_param_in_uri('human'),
split_lines=True), 200
def join():
global fetched_catalog
if get_jwt_identity() == '00000000-0000-0000-0000-000000000004':
js = request.get_json()
current_app.logger.debug(
f"New server wanting to join: {json.dumps(js, indent=2)}")
if db.session.query(Server).filter_by(
id=js.get('id', None)).count() > 0:
raise errors.DuplicatedId(js.get('id', None))
if db.session.query(Server).filter_by(
name=js.get('name', None)).count() > 0:
raise errors.AlreadyExists('name', js.get('name', None))
s = Server.from_json(js)
s.created_on = get_now()
external_ip = ipaddress.ip_address(request.remote_addr)
if not external_ip.is_loopback and external_ip not in [
gate.ip for gate in s.gates
]:
for port in set([gate.port for gate in s.gates]):
s.add_new_gate(external_ip, port, hidden=True)
certfile = current_app.dm.config.http_conf.get('certfile', None)
keyfile = current_app.dm.config.http_conf.get('keyfile', None)
if keyfile and os.path.exists(keyfile):
with open(keyfile, 'rb') as fh:
keyfile_content = fh.read()
else:
raise errors.FileNotFound(keyfile)
if certfile and os.path.exists(certfile):
with open(certfile, 'rb') as fh:
certfile_content = fh.read()
else:
raise errors.FileNotFound(certfile)
data = {
'keyfile': base64.b64encode(keyfile_content).decode(),
'certfile': base64.b64encode(certfile_content).decode()
}
data.update(Dimension=g.dimension.to_json())
data.update(me=str(Server.get_current().id))
with _lock:
if fetched_catalog[1] is None or fetched_catalog[0] < get_now(
) - dt.timedelta(minutes=1):
c = fetch_catalog()
fetched_catalog = (get_now(), c)
else:
c = fetched_catalog[1]
data.update(catalog=c)
if _lock_delete.acquire(False):
try:
delete_old_temp_servers()
finally:
_lock_delete.release()
server_data = s.to_json(add_gates=True)
servers_to_be_created.update({s.id: server_data})
del s
return data, 200
else:
raise errors.GenericError('Invalid token', status_code=400)
def _notify_cluster_in(self):
from dimensigon.domain.entities import Server
import dimensigon.web.network as ntwrk
from dimensigon.domain.entities import Parameter
try:
signaled = self._route_initiated.wait(timeout=120)
except Exception:
return
if not signaled:
self.logger.warning("Route Event not fired.")
self.logger.debug("Notify Cluster")
with self.dm.flask_app.app_context():
not_notify = set()
me = Server.get_current()
msg = [
r.to_json() for r in Route.query.options(
orm.lazyload(Route.destination), orm.lazyload(Route.gate),
orm.lazyload(Route.proxy_server)).all()
]
neighbours = Server.get_neighbours()
if Parameter.get('join_server', None):
join_server = Server.query.get(Parameter.get('join_server'))
else:
join_server = None
now = get_now()
msg = dict(keepalive=now.strftime(defaults.DATEMARK_FORMAT),
routes=msg)
if neighbours:
random.shuffle(neighbours)
first = [
s for s in neighbours
if s.id == Parameter.get('new_gates_server', None)
]
if first:
neighbours.pop(neighbours.index(first[0]))
neighbours = first + neighbours
elif join_server in neighbours:
neighbours.pop(neighbours.index(join_server))
neighbours = [join_server] + neighbours
for s in neighbours:
if s.id not in not_notify:
self.logger.debug(
f"Sending 'Cluster IN' message to {s}")
resp = ntwrk.post(s,
'api_1_0.cluster_in',
view_data=dict(server_id=str(me.id)),
json=msg,
timeout=10,
auth=get_root_auth())
if resp.ok:
converted = []
for ident, str_keepalive, death in resp.msg[
'cluster']:
try:
keepalive = dt.datetime.strptime(
str_keepalive,
defaults.DATEMARK_FORMAT)
except ValueError:
continue
converted.append((ident, keepalive, death))
self.put_many(converted)
not_notify.update(resp.msg.get('neighbours', []))
else:
self.logger.debug(
f"Unable to send 'Cluster IN' message to {s} . Response: {resp}"
)
else:
self.logger.debug(
f"Skiping server {s} from sending 'Cluster IN' message"
)
# alive = [(getattr(Server.query.get(s_id), 'name', None) or s_id) for s_id in
# self.get_alive()]
# self.logger.info(f"Alive servers: {', '.join(alive)}")
else:
self.logger.debug("No neighbour to send 'Cluster IN'")
self.logger.debug("Notify Cluster ended")
def create_cmd_from_orchestration(orchestration: Orchestration, var_context: Context,
hosts: t.Dict[str, t.List[Id]],
executor: concurrent.futures.Executor,
register: 'RegisterStepExecution') -> CompositeCommand:
current_server = Server.get_current()
def create_do_cmd_from_step(_step: Step, _s2cc):
d = {}
if _step in _s2cc:
cc = _s2cc[_step]
else:
activate_server_ctx = len([server_id for target in _step.target for server_id in hosts[target]]) > 1
for target in _step.target:
for server_id in hosts[target]:
if str(server_id) == str(current_server.id):
cls = Command
else:
cls = partial(ProxyCommand, server_id)
var_kwargs = dict()
if activate_server_ctx:
var_kwargs.update(key_server_ctx=server_id)
c = cls(create_operation(_step),
undo_command=_create_server_undo_command(executor, current_server, server_id,
_step, var_context, register),
var_context=var_context.local_ctx({'server_id': server_id}, **var_kwargs),
stop_on_error=_step.stop_on_error,
stop_undo_on_error=None,
undo_on_error=_step.undo_on_error,
pre_process=_step.pre_process,
post_process=_step.post_process,
register=register,
id_=(str(server_id), str(_step.id)),
signature=_step.schema)
d[c] = []
if len(d) == 1:
cc = c
elif len(d) > 1:
cc = CompositeCommand(dict_tree=d,
stop_on_error=False,
stop_undo_on_error=False,
id_=str(_step.id), executor=executor, register=register, var_context=var_context)
else:
cc = None
return cc
def iterate_tree(_step: Step, _d, _s2cc):
if _step in s2cc_map:
_c = _s2cc[_step]
else:
_c = create_do_cmd_from_step(_step, _s2cc)
_s2cc[_step] = _c
if _c not in _d:
_d[_c] = []
for child_step in _step.children_do_steps:
_d[_c].append(iterate_tree(child_step, _d, _s2cc))
return _c
root_steps = orchestration.root
tree = {}
s2cc_map = {}
for step in root_steps:
iterate_tree(step, tree, s2cc_map)
return CompositeCommand(dict_tree=tree,
stop_undo_on_error=orchestration.stop_undo_on_error,
stop_on_error=orchestration.stop_on_error,
id_=str(orchestration.id), executor=executor, register=register)