async def request_locker(servers: t.Union[Server, t.List[Server]],
action,
scope,
applicant,
auth=None,
datemark=None) -> t.List[Response]:
tasks = []
server_responses = []
if is_iterable_not_string(servers):
it = servers
else:
it = [servers]
payload = dict(scope=scope.name, applicant=applicant)
if datemark:
payload.update(datemark=datemark)
for server in it:
tasks.append(
create_task(
async_post(server,
'api_1_0.locker_' + action,
json=payload,
auth=auth,
timeout=defaults.TIMEOUT_LOCK_REQUEST)))
for server in it:
r = await tasks.pop(0)
server_responses.append(r)
return server_responses
def test_post(self, m):
status = 200
msg = {'new': 'data'}
data = {'data': 'some data'}
def callback(request, **kwargs):
if isinstance(request, PreparedRequest):
self.assertDictEqual(data,
unpack_msg(json.loads(request.body)))
return status, {}, json.dumps(pack_msg(msg))
else:
self.assertDictEqual(data, unpack_msg(kwargs['json']))
return CallbackResult(status=status, payload=pack_msg(msg))
responses.add_callback(responses.POST, self.url, callback=callback)
m.post(self.url, callback=callback)
resp = post(self.server, 'home', json=data)
self.assertEqual(status, resp.code)
self.assertDictEqual(msg, resp.msg)
resp = run(async_post(self.server, 'home', json=data))
self.assertEqual(status, resp.code)
self.assertDictEqual(msg, resp.msg)
def test_post_no_content_in_response(self, m):
msg = ''
status = 204
responses.add(responses.POST, self.url, status=204)
m.post(self.url, status=204)
data, status = post(self.server, 'home')
self.assertEqual(status, status)
self.assertEqual(msg, data)
data, status = run(async_post(self.server, 'home'))
self.assertEqual(status, status)
self.assertEqual(msg, data)
async def _async_get_neighbour_healthcheck(self,
cluster_heartbeat_id: str = None
) -> t.Dict[Server, dict]:
server_responses = {}
servers = Server.get_neighbours()
self.logger.debug(
f"Neighbour servers to check: {', '.join([s.name for s in servers])}"
)
auth = get_root_auth()
if cluster_heartbeat_id is None:
cluster_heartbeat_id = get_now().strftime(defaults.DATETIME_FORMAT)
cos = [
ntwrk.async_post(server,
'root.healthcheck',
json={
'me': self.dm.server_id,
'heartbeat': cluster_heartbeat_id
},
auth=auth) for server in servers
]
responses = await asyncio.gather(*cos)
for server, resp in zip(servers, responses):
if resp.ok:
id_response = resp.msg.get('server', {}).get('id', '')
if id_response and str(server.id) != id_response:
e = HealthCheckMismatch(expected={
'id': str(server.id),
'name': server.name
},
actual=resp.msg.get('server', {}))
self.logger.warning(str(e))
else:
server_responses.update({server: resp.msg})
else:
self.logger.warning(
f"Unable to get Healthcheck from server {server.name}: {resp}"
)
return server_responses
def _send_new_data(self):
self.update_mapper()
tasks = OrderedDict()
for log_id, pb in self._mapper.items():
log = self.session.query(Log).get(log_id)
for pytail in pb:
data = pytail.fetch()
data = data.encode() if isinstance(data, str) else data
if data and log.destination_server.id in self.dm.cluster_manager.get_alive(
):
if log.mode == Mode.MIRROR:
file = pytail.file
elif log.mode == Mode.REPO_ROOT:
path_to_remove = os.path.dirname(log.target)
relative = os.path.relpath(pytail.file, path_to_remove)
file = os.path.join('{LOG_REPO}', relative)
elif log.mode == Mode.FOLDER:
path_to_remove = os.path.dirname(log.target)
relative = os.path.relpath(pytail.file, path_to_remove)
file = os.path.join(log.dest_folder, relative)
else:
def get_root(dirname):
new_dirname = os.path.dirname(dirname)
if new_dirname == dirname:
return dirname
else:
return get_root(new_dirname)
relative = os.path.relpath(pytail.file,
get_root(pytail.file))
file = os.path.join('{LOG_REPO}', relative)
with self.dm.flask_app.app_context():
auth = get_root_auth()
task = ntwrk.async_post(
log.destination_server,
'api_1_0.logresource',
view_data={'log_id': str(log_id)},
json={
"file":
file,
'data':
base64.b64encode(
zlib.compress(data)).decode('ascii'),
"compress":
True
},
auth=auth)
tasks[task] = (pytail, log)
_log_logger.debug(
f"Task sending data from '{pytail.file}' to '{log.destination_server}' prepared"
)
if tasks:
with self.dm.flask_app.app_context():
responses = asyncio.run(asyncio.gather(*list(tasks.keys())))
for task, resp in zip(tasks.keys(), responses):
pytail, log = tasks[task]
if resp.ok:
pytail.update_offset_file()
_log_logger.debug(f"Updated offset from '{pytail.file}'")
if log.id not in self._blacklist:
self._blacklist_log.pop(log.id, None)
else:
_log_logger.error(
f"Unable to send log information from '{pytail.file}' to '{log.destination_server}'. Error: {resp}"
)
if log.id not in self._blacklist:
bl = BlacklistEntry()
self._blacklist_log[log.id] = bl
else:
bl = self._blacklist_log.get(log.id)
bl.retries += 1
if bl.retries >= self.max_allowed_errors:
_log_logger.debug(
f"Adding server {log.destination_server.id} to the blacklist."
)
bl.blacklisted = time.time()
async def _send_file(self, file: File, servers: t.List[Id] = None):
try:
content = await self._loop.run_in_executor(self._executor,
self._read_file,
file.target)
except Exception as e:
self.logger.exception(
f"Unable to get content from file {file.target}.")
return
if servers:
server_ids = servers
fsas = [
fsa for fsa in file.destinations
if fsa.destination_server.id in server_ids
]
else:
server_ids = [
fsa.destination_server.id for fsa in file.destinations
]
fsas = file.destinations
with self.dm.flask_app.app_context():
auth = get_root_auth()
alive = self.dm.cluster_manager.get_alive()
tasks = [
ntwrk.async_post(fsa.destination_server,
view_or_url='api_1_0.file_sync',
view_data={'file_id': file.id},
json=dict(file=fsa.target,
data=content,
force=True),
auth=auth) for fsa in fsas
if fsa.destination_server.id in alive
]
skipped = [
fsa.destination_server.name for fsa in fsas
if fsa.destination_server.id not in alive
]
if skipped:
self.logger.debug(
f"Following servers are skipped because we do not see them alive: {', '.join(skipped)}"
)
if tasks:
self.logger.debug(
f"Syncing file {file} with the following servers: {', '.join([fsa.destination_server.name for fsa in fsas if fsa.destination_server.id in alive])}."
)
resp = await asyncio.gather(*tasks)
for resp, fsa in zip(resp, fsas):
if not resp.ok:
self.logger.warning(
f"Unable to send file {file.target} to {fsa.destination_server}. Reason: {resp}"
)
if (file.id, fsa.destination_server.id
) not in self._blacklist:
bl = BlacklistEntry()
self._blacklist[(file.id,
fsa.destination_server.id)] = bl
else:
bl = self._blacklist.get(
(file.id, fsa.destination_server.id))
bl.retries += 1
if bl.retries >= self.max_allowed_errors:
self.logger.debug(
f"Adding server {fsa.destination_server} to the blacklist."
)
bl.blacklisted = time.time()
else:
if (file.id,
fsa.destination_server.id) in self._blacklist:
self._blacklist.pop(
(file.id, fsa.destination_server.id), None)
fsa.l_mtime = file.l_mtime
try:
self.session.commit()
except:
self.session.rollback()