async def send_request(self, command: bytes, data: bytes) -> bytes:
"""
Sends a request to peer
:param command: command to send
:param data: data to send
:return: response from peer.
"""
if len(data) > self.request_chunk:
raise exception.WazuhClusterError(3033)
response = Response()
msg_counter = self.next_counter()
self.box[msg_counter] = response
try:
self.push(self.msg_build(command, msg_counter, data))
except MemoryError:
self.request_chunk //= 2
raise exception.WazuhClusterError(3026)
except Exception as e:
raise exception.WazuhClusterError(3018, extra_message=str(e))
try:
response_data = await asyncio.wait_for(response.read(),
timeout=self.cluster_items['intervals']['communication'][
'timeout_cluster_request'])
del self.box[msg_counter]
except asyncio.TimeoutError:
self.box[msg_counter] = None
return b'Error sending request: timeout expired.'
return response_data
def hello(self, data: bytes) -> Tuple[bytes, bytes]:
"""Add a client's data to global clients dictionary.
Parameters
----------
data : bytes
Client's name.
Returns
-------
bytes
Result.
bytes
Response message.
"""
self.name = data.decode()
if self.name in self.server.clients:
self.name = ''
raise exception.WazuhClusterError(3028, extra_message=data)
elif self.name == self.server.configuration['node_name']:
raise exception.WazuhClusterError(3029)
else:
self.server.clients[self.name] = self
self.tag = f'{self.tag} {self.name}'
context_tag.set(self.tag)
return b'ok', f'Client {self.name} added'.encode()
def hello(self, data: bytes) -> Tuple[bytes, bytes]:
"""
Processes "hello" command sent by a worker right after it connects to the server. It also initializes
the task loggers.
:param data: Node name, cluster name, node type and wazuh version all separated by spaces.
:return: response command and payload.
"""
name, cluster_name, node_type, version = data.split(b' ')
cmd, payload = super().hello(name)
self.task_loggers = {'Integrity': self.setup_task_logger('Integrity'),
'Extra valid': self.setup_task_logger('Extra valid')}
self.version, self.cluster_name, self.node_type = version.decode(), cluster_name.decode(), node_type.decode()
if self.cluster_name != self.server.configuration['name']:
raise exception.WazuhClusterError(3030)
elif self.version != metadata.__version__:
raise exception.WazuhClusterError(3031)
worker_dir = '{}/queue/cluster/{}'.format(common.ossec_path, self.name)
if cmd == b'ok' and not os.path.exists(worker_dir):
utils.mkdir_with_mode(worker_dir)
return cmd, payload
async def sync(self):
"""Start synchronization process with the master and send necessary information."""
start_time = time.time()
result = await self.worker.send_request(command=self.cmd+b'_p', data=b'')
if isinstance(result, Exception):
self.logger.error(f"Error asking for permission: {result}")
return
elif result == b'True':
self.logger.debug("Permission to synchronize granted.")
else:
self.logger.info(f"Finished in {(time.time()-start_time):.3f}s. Master didn't grant permission to "
f"synchronize.")
return
# Start the synchronization process with the master and get a taskID.
task_id = await self.worker.send_request(command=self.cmd, data=b'')
if isinstance(task_id, Exception):
raise task_id
elif task_id.startswith(b'Error'):
self.logger.error(task_id.decode())
exc_info = json.dumps(exception.WazuhClusterError(code=3016, extra_message=str(task_id)),
cls=c_common.WazuhJSONEncoder).encode()
await self.worker.send_request(command=self.cmd + b'_r', data=b'None ' + exc_info)
return
self.logger.debug("Compressing files and 'files_metadata.json'." if self.files_to_sync else
"Compressing 'files_metadata.json'.")
compressed_data_path = wazuh.core.cluster.cluster.compress_files(name=self.worker.name,
list_path=self.files_to_sync,
cluster_control_json=self.files_metadata)
try:
# Send zip file to the master into chunks.
self.logger.debug("Sending zip file to master.")
await self.worker.send_file(filename=compressed_data_path)
self.logger.debug("Zip file sent to master.")
# Finish the synchronization process and notify where the file corresponding to the taskID is located.
result = await self.worker.send_request(command=self.cmd + b'_e', data=task_id + b' ' +
os.path.relpath(compressed_data_path, common.wazuh_path).encode())
if isinstance(result, Exception):
raise result
elif result.startswith(b'Error'):
raise WazuhClusterError(3016, extra_message=result.decode())
return True
except exception.WazuhException as e:
# Notify error to master and delete its received file.
self.logger.error(f"Error sending zip file: {e}")
await self.worker.send_request(command=self.cmd+b'_r', data=task_id + b' ' +
json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
# Notify error to master and delete its received file.
self.logger.error(f"Error sending zip file: {e}")
exc_info = json.dumps(exception.WazuhClusterError(code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder).encode()
await self.worker.send_request(command=self.cmd+b'_r', data=task_id + b' ' + exc_info)
finally:
os.unlink(compressed_data_path)
async def execute(self, command: bytes, data: bytes, wait_for_complete: bool) -> Dict:
"""Send DAPI request and wait for response.
Send a distributed API request and wait for a response in command dapi_res. Methods here are the same
as the ones defined in LocalServerHandlerMaster.
Parameters
----------
command : bytes
Command to execute.
data : bytes
Data to send.
wait_for_complete : bool
Whether to raise a timeout exception or not.
Returns
-------
request_result : dict
API response.
"""
request_id = str(random.randint(0, 2**10 - 1))
# Create an event to wait for the response.
self.server.pending_api_requests[request_id] = {'Event': asyncio.Event(), 'Response': ''}
# If forward request to other worker, get destination client and request.
if command == b'dapi_forward':
client, request = data.split(b' ', 1)
client = client.decode()
if client in self.server.clients:
result = (await self.server.clients[client].send_request(b'dapi', request_id.encode() + b' ' + request)).decode()
else:
raise exception.WazuhClusterError(3022, extra_message=client)
# Add request to local API requests queue.
elif command == b'dapi':
result = (await self.send_request(b'dapi', request_id.encode() + b' ' + data)).decode()
# If not dapi related command, run it now.
else:
result = self.process_request(command=command, data=data)
# If command was dapi or dapi_forward, wait for response.
if command == b'dapi' or command == b'dapi_forward':
try:
timeout = None if wait_for_complete \
else self.cluster_items['intervals']['communication']['timeout_api_request']
await asyncio.wait_for(self.server.pending_api_requests[request_id]['Event'].wait(), timeout=timeout)
request_result = self.server.pending_api_requests[request_id]['Response']
except asyncio.TimeoutError:
raise exception.WazuhClusterError(3021)
# Otherwise, immediately return the result obtained before.
else:
status, request_result = result
if status != b'ok':
raise exception.WazuhClusterError(3022, extra_message=request_result.decode())
request_result = request_result.decode()
return request_result
async def sync_wazuh_db_info(self, task_id: bytes):
"""Iterate and update in the local wazuh-db the chunks of data received from a worker.
Parameters
----------
task_id : bytes
ID of the string where the JSON chunks are stored.
Returns
-------
result : bytes
Worker's response after finishing the synchronization.
"""
logger = self.task_loggers['Agent-info sync']
logger.info(f"Starting")
date_start_master = datetime.now()
wdb_conn = WazuhDBConnection()
result = {'updated_chunks': 0, 'error_messages': list()}
try:
# Chunks were stored under 'task_id' as an string.
received_string = self.in_str[task_id].payload
data = json.loads(received_string.decode())
except KeyError as e:
await self.send_request(command=b'syn_m_a_err',
data=f"error while trying to access string under task_id {str(e)}.".encode())
raise exception.WazuhClusterError(3035, extra_message=f"it should be under task_id {str(e)}, but it's empty.")
except ValueError as e:
await self.send_request(command=b'syn_m_a_err', data=f"error while trying to load JSON: {str(e)}".encode())
raise exception.WazuhClusterError(3036, extra_message=str(e))
# Update chunks in local wazuh-db
before = time()
for i, chunk in enumerate(data['chunks']):
try:
logger.debug2(f"Sending chunk {i+1}/{len(data['chunks'])} to wazuh-db: {chunk}")
response = wdb_conn.send(f"{data['set_data_command']} {chunk}", raw=True)
if response[0] != 'ok':
result['error_messages'].append(response)
logger.error(f"Response for chunk {i}/{len(data['chunks'])} was not 'ok': {response}")
else:
result['updated_chunks'] += 1
except Exception as e:
result['error_messages'].append(str(e))
logger.debug(f"All chunks updated in wazuh-db in {(time() - before):3f}s.")
# Send result to worker
response = await self.send_request(command=b'syn_m_a_e', data=json.dumps(result).encode())
self.sync_agent_info_status.update({'date_start_master': date_start_master, 'date_end_master': datetime.now(),
'n_synced_chunks': result['updated_chunks']})
logger.info("Finished in {:.3f}s ({} chunks updated).".format((self.sync_agent_info_status['date_end_master'] -
self.sync_agent_info_status['date_start_master'])
.total_seconds(), result['updated_chunks']))
return response
async def execute(self, command: bytes, data: bytes,
wait_for_complete: bool) -> Dict:
"""
Sends a distributed API request and wait for a response in command dapi_res. Methods here are the same
as the ones defined in LocalServerHandlerMaster.
:param command: Command to execute
:param data: Data to send
:param wait_for_complete: Raise a timeout exception or not
:return: A dictionary with the API response
"""
request_id = str(random.randint(0, 2**10 - 1))
# create an event to wait for the response
self.server.pending_api_requests[request_id] = {
'Event': asyncio.Event(),
'Response': ''
}
if command == b'dapi_forward':
client, request = data.split(b' ', 1)
client = client.decode()
if client in self.server.clients:
result = (await self.server.clients[client].send_request(
b'dapi',
request_id.encode() + b' ' + request)).decode()
else:
raise exception.WazuhClusterError(3022, extra_message=client)
elif command == b'dapi':
result = (await self.send_request(
b'dapi',
request_id.encode() + b' ' + data)).decode()
else:
result = self.process_request(command=command, data=data)
if command == b'dapi' or command == b'dapi_forward':
try:
timeout = None if wait_for_complete \
else self.cluster_items['intervals']['communication']['timeout_api_request']
await asyncio.wait_for(
self.server.pending_api_requests[request_id]
['Event'].wait(),
timeout=timeout)
request_result = self.server.pending_api_requests[request_id][
'Response']
except asyncio.TimeoutError:
raise exception.WazuhClusterError(3021)
else:
status, request_result = result
if status != b'ok':
raise exception.WazuhClusterError(
3022, extra_message=request_result.decode())
request_result = request_result.decode()
return request_result
async def sync(self):
"""
Start synchronization process with the master and send necessary information
"""
result = await self.worker.send_request(command=self.cmd+b'_p', data=b'')
if isinstance(result, Exception):
self.logger.error(f"Error asking for permission: {result}")
return
elif result == b'True':
self.logger.info("Permission to synchronize granted")
else:
self.logger.info('Master didnt grant permission to synchronize')
return
task_id = await self.worker.send_request(command=self.cmd, data=b'')
if isinstance(task_id, Exception):
raise task_id
elif task_id.startswith(b'Error'):
self.logger.error(task_id.decode())
exc_info = json.dumps(exception.WazuhClusterError(code=3016, extra_message=str(task_id)),
cls=c_common.WazuhJSONEncoder).encode()
await self.worker.send_request(command=self.cmd + b'_r', data=b'None ' + exc_info)
return
self.logger.info("Compressing files")
compressed_data_path = wazuh.core.cluster.cluster.compress_files(name=self.worker.name,
list_path=self.files_to_sync,
cluster_control_json=self.checksums)
try:
self.logger.info("Sending compressed file to master")
await self.worker.send_file(filename=compressed_data_path)
self.logger.info("Worker files sent to master")
result = await self.worker.send_request(command=self.cmd + b'_e', data=task_id + b' ' +
os.path.relpath(compressed_data_path, common.ossec_path).encode())
if isinstance(result, Exception):
raise result
elif result.startswith(b'Error'):
raise WazuhClusterError(3016, extra_message=result.decode())
except exception.WazuhException as e:
self.logger.error(f"Error sending files information: {e}")
await self.worker.send_request(command=self.cmd+b'_r', data=task_id + b' ' +
json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
self.logger.error(f"Error sending files information: {e}")
exc_info = json.dumps(exception.WazuhClusterError(code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder).encode()
await self.worker.send_request(command=self.cmd+b'_r', data=task_id + b' ' + exc_info)
finally:
os.unlink(compressed_data_path)
def get_messages(self) -> Tuple[bytes, int, bytes]:
"""
Called when data is received in the transport. It decrypts the received data and returns it using generators.
If the data received in the transport contains multiple separated messages, it will return all of them in
separate yields.
:return: Last received message command, counter and payload
"""
parsed = self.msg_parse()
while parsed:
# self.logger.debug("Received message: {} / {}".format(self.in_msg['received'], self.in_msg['total_size']))
if self.in_msg.received == self.in_msg.total:
# the message was correctly received
# decrypt received message
try:
decrypted_payload = self.my_fernet.decrypt(bytes(self.in_msg.payload)) if self.my_fernet is not None \
else bytes(self.in_msg.payload)
except cryptography.fernet.InvalidToken:
raise exception.WazuhClusterError(3025)
yield self.in_msg.cmd, self.in_msg.counter, decrypted_payload
self.in_msg = InBuffer()
else:
break
parsed = self.msg_parse()
async def forward_dapi_response(self, data: bytes):
"""Forward a distributed API response from master node.
Parameters
----------
data : bytes
Bytes containing local client name and string id separated by ' '.
"""
client, string_id = data.split(b' ', 1)
client = client.decode()
try:
res = await self.get_manager(
).local_server.clients[client].send_string(
self.in_str[string_id].payload)
res = await self.get_manager(
).local_server.clients[client].send_request(b'dapi_res', res)
except exception.WazuhException as e:
self.logger.error(
f"Error sending API response to local client: {e}")
res = await self.send_request(
b'dapi_err',
json.dumps(e, cls=WazuhJSONEncoder).encode())
except Exception as e:
self.logger.error(
f"Error sending API response to local client: {e}")
exc_info = json.dumps(exception.WazuhClusterError(
code=1000, extra_message=str(e)),
cls=WazuhJSONEncoder).encode()
res = await self.send_request(b'dapi_err', exc_info)
finally:
# Remove the string after using it
self.in_str.pop(string_id, None)
async def send_file(self, filename: str) -> bytes:
"""
Sends a file to peer.
:param filename: File path to send
:return: response message.
"""
if not os.path.exists(filename):
raise exception.WazuhClusterError(3034, extra_message=filename)
filename = filename.encode()
relative_path = filename.replace(common.ossec_path.encode(), b'')
response = await self.send_request(command=b'new_file',
data=relative_path)
file_hash = hashlib.sha256()
with open(filename, 'rb') as f:
for chunk in iter(
lambda: f.read(self.request_chunk - len(relative_path) - 1
), b''):
response = await self.send_request(command=b'file_upd',
data=relative_path + b' ' +
chunk)
file_hash.update(chunk)
response = await self.send_request(command=b'file_end',
data=relative_path + b' ' +
file_hash.digest())
return b'File sent'
def msg_build(self, command: bytes, counter: int, data: bytes) -> bytes:
"""
Builds a message with header + payload
:param command: command to send
:param counter: message id
:param data: data to send
:return: built message
"""
cmd_len = len(command)
if cmd_len > self.cmd_len:
raise exception.WazuhClusterError(3024, extra_message=command)
# adds - to command until it reaches cmd length
command = command + b' ' + b'-' * (self.cmd_len - cmd_len - 1)
encrypted_data = self.my_fernet.encrypt(
data) if self.my_fernet is not None else data
self.out_msg[:self.header_len] = struct.pack(self.header_format,
counter,
len(encrypted_data),
command)
self.out_msg[self.header_len:self.header_len +
len(encrypted_data)] = encrypted_data
return self.out_msg[:self.header_len + len(encrypted_data)]
def end_receiving_file(self,
task_and_file_names: str) -> Tuple[bytes, bytes]:
"""Store full path to the received file in task_name and notify its availability.
Parameters
----------
task_and_file_names : str
String containing task ID and relative filepath separated by ' '.
Returns
-------
bytes
Result.
bytes
Response message.
"""
task_name, filename = task_and_file_names.split(' ', 1)
if task_name not in self.sync_tasks:
# Remove filename if task_name does not exists, before raising exception.
if os.path.exists(os.path.join(common.wazuh_path, filename)):
try:
os.remove(os.path.join(common.wazuh_path, filename))
except Exception as e:
self.get_logger(self.logger_tag).error(
f"Attempt to delete file {os.path.join(common.wazuh_path, filename)} failed: {e}"
)
raise exception.WazuhClusterError(3027, extra_message=task_name)
# Set full path to file for task 'task_name' and notify it is ready to be read, so the lock is released.
self.sync_tasks[task_name].filename = os.path.join(
common.wazuh_path, filename)
self.sync_tasks[task_name].received_information.set()
return b'ok', b'File correctly received'
async def sync_extra_valid(self, extra_valid: Dict):
"""
Asynchronous task that is started when the master requests any extra valid files to be synchronized.
That means, it is started in the sync_integrity process.
:param extra_valid: Files required by the master
:return: None
"""
extra_valid_logger = self.task_loggers["Extra valid"]
try:
before = time.time()
self.logger.debug("Starting to send extra valid files")
# TODO: Add support for more extra valid file types if ever added
n_files, merged_file = wazuh.core.cluster.cluster.merge_agent_info(merge_type='agent-groups', files=extra_valid.keys(),
time_limit_seconds=0, node_name=self.name)
if n_files:
files_to_sync = {merged_file: {'merged': True, 'merge_type': 'agent-groups', 'merge_name': merged_file,
'cluster_item_key': '/queue/agent-groups/'}}
my_worker = SyncWorker(cmd=b'sync_e_w_m', files_to_sync=files_to_sync, checksums=files_to_sync,
logger=extra_valid_logger, worker=self)
await my_worker.sync()
after = time.time()
self.logger.debug2("Time synchronizing extra valid files: {} s".format(after - before))
except exception.WazuhException as e:
extra_valid_logger.error("Error synchronizing extra valid files: {}".format(e))
res = await self.send_request(command=b'sync_e_w_m_r',
data=b'None ' + json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
extra_valid_logger.error("Error synchronizing extra valid files: {}".format(e))
exc_info = json.dumps(exception.WazuhClusterError(code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder)
res = await self.send_request(command=b'sync_e_w_m_r', data=b'None ' + exc_info.encode())
async def sync_integrity(self):
"""
Asynchronous task that is started when the worker connects to the master. It starts an integrity synchronization
process every self.cluster_items['intervals']['worker']['sync_integrity'] seconds.
:return: None
"""
integrity_logger = self.task_loggers["Integrity"]
while True:
try:
if self.connected:
before = time.time()
await SyncWorker(cmd=b'sync_i_w_m', files_to_sync={}, checksums=wazuh.core.cluster.cluster.get_files_status('master',
self.name),
logger=integrity_logger, worker=self).sync()
after = time.time()
integrity_logger.debug("Time synchronizing integrity: {} s".format(after - before))
except exception.WazuhException as e:
integrity_logger.error("Error synchronizing integrity: {}".format(e))
res = await self.send_request(command=b'sync_i_w_m_r',
data=json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
integrity_logger.error("Error synchronizing integrity: {}".format(e))
exc_info = json.dumps(exception.WazuhClusterError(code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder)
res = await self.send_request(command=b'sync_i_w_m_r', data=exc_info.encode())
await asyncio.sleep(self.cluster_items['intervals']['worker']['sync_integrity'])
async def send_file(self, filename: str) -> bytes:
"""Send a file to peer, slicing it into chunks.
Parameters
----------
filename : str
Full path of the file to send.
Returns
-------
bytes
Response message.
"""
if not os.path.exists(filename):
raise exception.WazuhClusterError(3034, extra_message=filename)
filename = filename.encode()
relative_path = filename.replace(common.wazuh_path.encode(), b'')
# Tell to the destination node where (inside wazuh_path) the file has to be written.
await self.send_request(command=b'new_file', data=relative_path)
# Send each chunk so it is updated in the destination.
file_hash = hashlib.sha256()
with open(filename, 'rb') as f:
data = f.read()
await self.send_request(command=b'file_upd',
data=relative_path + b' ' + data)
file_hash.update(data)
# Close the destination file descriptor so the file in memory is dumped to disk.
await self.send_request(command=b'file_end',
data=relative_path + b' ' + file_hash.digest())
return b'File sent'
def msg_build(self, command: bytes, counter: int, data: bytes) -> bytes:
"""Build a message with header + payload.
It contains a header in self.header_format format that includes self.counter, the data size and the command.
The data is also encrypted and added to the bytearray starting from the position self.header_len.
Parameters
----------
command : bytes
Command to send to peer.
counter : int
Message ID.
data : bytes
Data to send to peer.
Returns
-------
bytes
Built message.
"""
cmd_len = len(command)
if cmd_len > self.cmd_len:
raise exception.WazuhClusterError(3024, extra_message=command)
# adds - to command until it reaches cmd length
command = command + b' ' + b'-' * (self.cmd_len - cmd_len - 1)
encrypted_data = self.my_fernet.encrypt(
data) if self.my_fernet is not None else data
out_msg = bytearray(self.header_len + len(encrypted_data))
out_msg[:self.header_len] = struct.pack(self.header_format, counter,
len(encrypted_data), command)
out_msg[self.header_len:self.header_len +
len(encrypted_data)] = encrypted_data
return out_msg
def get_messages(self) -> Tuple[bytes, int, bytes]:
"""Get received command, counter and payload.
Called when data is received in the transport. It decrypts the received data and returns it using generators.
If the data received in the transport contains multiple separated messages, it will return all of them in
separate yields.
Yields
-------
bytes
Last received message command.
int
Counter.
bytes
Payload.
"""
parsed = self.msg_parse()
while parsed:
if self.in_msg.received == self.in_msg.total:
# Decrypt received message
try:
decrypted_payload = self.my_fernet.decrypt(bytes(self.in_msg.payload)) if self.my_fernet is not None \
else bytes(self.in_msg.payload)
except cryptography.fernet.InvalidToken:
raise exception.WazuhClusterError(3025)
yield self.in_msg.cmd, self.in_msg.counter, decrypted_payload
self.in_msg = InBuffer()
else:
break
parsed = self.msg_parse()
async def forward_sendsync_response(self, data: bytes):
"""Forward a sendsync response from master node.
Parameters
----------
data : bytes
Bytes containing local client name and string id separated by ' '.
"""
client, string_id = data.split(b' ', 1)
client = client.decode()
try:
await self.get_manager().local_server.clients[client].send_request(
b'ok', self.in_str[string_id].payload)
except exception.WazuhException as e:
self.logger.error(
f"Error sending send sync response to local client: {e}")
await self.send_request(
b'sendsync_err',
json.dumps(e, cls=WazuhJSONEncoder).encode())
except Exception as e:
self.logger.error(
f"Error sending send sync response to local client: {e}")
exc_info = json.dumps(exception.WazuhClusterError(
code=1000, extra_message=str(e)),
cls=WazuhJSONEncoder).encode()
await self.send_request(b'sendsync_err', exc_info)
def process_dapi_res(self, data: bytes) -> Tuple[bytes, bytes]:
"""Process a DAPI response coming from a worker node.
This function is called when the master received a "dapi_res" command. The response
has been previously sent using a send_string so this method only receives the string ID.
If the request ID is within the pending api requests, the response is assigned to the request ID and
the server is notified. Else, if the request ID is within the local_server clients, it is forwarded.
Parameters
----------
data : bytes
Request ID and response ID separated by a space (' ').
Returns
-------
bytes
Result.
bytes
Response message.
"""
req_id, string_id = data.split(b' ', 1)
req_id = req_id.decode()
if req_id in self.server.pending_api_requests:
self.server.pending_api_requests[req_id]['Response'] = self.in_str[
string_id].payload.decode()
self.server.pending_api_requests[req_id]['Event'].set()
return b'ok', b'Forwarded response'
elif req_id in self.server.local_server.clients:
asyncio.create_task(self.forward_dapi_response(data))
return b'ok', b'Response forwarded to worker'
else:
raise exception.WazuhClusterError(3032, extra_message=req_id)
async def sync_integrity(self):
"""Obtain files status and send it to the master.
Asynchronous task that is started when the worker connects to the master. It starts an integrity synchronization
process every self.cluster_items['intervals']['worker']['sync_integrity'] seconds.
A dictionary like {'file_path': {<MD5, merged, merged_name, etc>}, ...} is created and sent to the master,
containing the information of all the files inside the directories specified in cluster.json. The master
compares it with its own information.
"""
logger = self.task_loggers["Integrity check"]
while True:
try:
if self.connected:
logger.info("Starting.")
start_time = time.time()
if await SyncWorker(cmd=b'sync_i_w_m', files_to_sync={}, logger=logger, worker=self,
files_metadata=wazuh.core.cluster.cluster.get_files_status()).sync():
self.integrity_check_status['date_start'] = start_time
# If exception is raised during sync process, notify the master so it removes the file if received.
except exception.WazuhException as e:
logger.error(f"Error synchronizing integrity: {e}")
await self.send_request(command=b'sync_i_w_m_r', data=b'None ' +
json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
logger.error(f"Error synchronizing integrity: {e}")
exc_info = json.dumps(exception.WazuhClusterError(code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder)
await self.send_request(command=b'sync_i_w_m_r', data=b'None ' + exc_info.encode())
await asyncio.sleep(self.cluster_items['intervals']['worker']['sync_integrity'])
async def sync_extra_valid(self, extra_valid: Dict):
"""Merge and send files of the worker node that are missing in the master node.
Asynchronous task that is started when the master requests any extra valid files to be synchronized.
That means, it is started in the sync_integrity process.
Parameters
----------
extra_valid : dict
Keys are paths of files missing in the master node.
"""
logger = self.task_loggers["Integrity sync"]
try:
before = time.time()
logger.debug("Starting sending extra valid files to master.")
extra_valid_sync = SyncFiles(cmd=b'syn_e_w_m',
logger=logger,
worker=self)
# Merge all agent-groups files into one and create metadata dict with it (key->filepath, value->metadata).
n_files, merged_file = wazuh.core.cluster.cluster.merge_info(
merge_type='agent-groups',
node_name=self.name,
files=extra_valid.keys())
files_to_sync = {
merged_file: {
'merged': True,
'merge_type': 'agent-groups',
'merge_name': merged_file,
'cluster_item_key': 'queue/agent-groups/'
}
} if n_files else {}
# Permission is not requested since it was already granted in the 'Integrity check' task.
await extra_valid_sync.sync(files_to_sync=files_to_sync,
files_metadata=files_to_sync)
after = time.time()
logger.debug(
f"Finished sending extra valid files in {(after - before):.3f}s."
)
logger.info(
f"Finished in {(after - self.integrity_sync_status['date_start']):.3f}s."
)
# If exception is raised during sync process, notify the master so it removes the file if received.
except exception.WazuhException as e:
logger.error(f"Error synchronizing extra valid files: {e}")
await self.send_request(
command=b'syn_i_w_m_r',
data=b'None ' +
json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
logger.error(f"Error synchronizing extra valid files: {e}")
exc_info = json.dumps(exception.WazuhClusterError(
code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder)
await self.send_request(command=b'syn_i_w_m_r',
data=b'None ' + exc_info.encode())
def end_receiving_file(self, task_and_file_names: str) -> Tuple[bytes, bytes]:
task_name, filename = task_and_file_names.split(' ', 1)
if task_name not in self.sync_tasks:
raise exception.WazuhClusterError(3027, extra_message=task_name)
self.sync_tasks[task_name].filename = os.path.join(common.ossec_path, filename)
self.sync_tasks[task_name].received_information.set()
return b'ok', b'File correctly received'
def hello(self, data: bytes) -> Tuple[bytes, bytes]:
"""
Adds a client's data to global clients dictionary
:param data: client's data -> name
:return: successful result
"""
self.name = data.decode()
if self.name in self.server.clients:
self.name = ''
raise exception.WazuhClusterError(3028, extra_message=data)
elif self.name == self.server.configuration['node_name']:
raise exception.WazuhClusterError(3029)
else:
self.server.clients[self.name] = self
self.tag = '{} {}'.format(self.tag, self.name)
context_tag.set(self.tag)
return b'ok', 'Client {} added'.format(self.name).encode()
def hello(self, data: bytes) -> Tuple[bytes, bytes]:
"""Process 'hello' command from worker.
Process 'hello' command sent by a worker right after it connects to the server. It also initializes
the task loggers.
Parameters
----------
data : bytes
Node name, cluster name, node type and wazuh version all separated by spaces.
Returns
-------
cmd : bytes
Result.
payload : bytes
Response message.
"""
name, cluster_name, node_type, version = data.split(b' ')
# Add client to global clients dictionary.
cmd, payload = super().hello(name)
self.task_loggers = {
'Integrity check': self.setup_task_logger('Integrity check'),
'Integrity sync': self.setup_task_logger('Integrity sync'),
'Agent-info sync': self.setup_task_logger('Agent-info sync')
}
# Fill more information and check both name and version are correct.
self.version, self.cluster_name, self.node_type = version.decode(
), cluster_name.decode(), node_type.decode()
if self.cluster_name != self.server.configuration['name']:
raise exception.WazuhClusterError(3030)
elif self.version != metadata.__version__:
raise exception.WazuhClusterError(3031)
# Create directory where zips and other files coming from or going to the worker will be managed.
worker_dir = os.path.join(common.wazuh_path, 'queue', 'cluster',
self.name)
if cmd == b'ok' and not os.path.exists(worker_dir):
utils.mkdir_with_mode(worker_dir)
return cmd, payload
async def sync_extra_valid(self, extra_valid: Dict):
"""Merge and send files of the worker node that are missing in the master node.
Asynchronous task that is started when the master requests any extra valid files to be synchronized.
That means, it is started in the sync_integrity process.
Parameters
----------
extra_valid : dict
Keys are paths of files missing in the master node.
"""
extra_valid_logger = self.task_loggers["Extra valid"]
try:
before = time.time()
self.logger.debug("Starting to send extra valid files")
# Merge all agent-groups files into one and create checksums dict with it (key->filepath, value->metadata).
n_files, merged_file = \
wazuh.core.cluster.cluster.merge_info(merge_type='agent-groups', files=extra_valid.keys(),
time_limit_seconds=0, node_name=self.name)
if n_files:
files_to_sync = {
merged_file: {
'merged': True,
'merge_type': 'agent-groups',
'merge_name': merged_file,
'cluster_item_key': '/queue/agent-groups/'
}
}
my_worker = SyncWorker(cmd=b'sync_e_w_m',
files_to_sync=files_to_sync,
checksums=files_to_sync,
logger=extra_valid_logger,
worker=self)
await my_worker.sync()
after = time.time()
self.logger.debug2(
"Time synchronizing extra valid files: {} s".format(after -
before))
# If exception is raised during sync process, notify the master so it removes the file if received.
except exception.WazuhException as e:
extra_valid_logger.error(
"Error synchronizing extra valid files: {}".format(e))
await self.send_request(
command=b'sync_e_w_m_r',
data=b'None ' +
json.dumps(e, cls=c_common.WazuhJSONEncoder).encode())
except Exception as e:
extra_valid_logger.error(
"Error synchronizing extra valid files: {}".format(e))
exc_info = json.dumps(exception.WazuhClusterError(
code=1000, extra_message=str(e)),
cls=c_common.WazuhJSONEncoder)
await self.send_request(command=b'sync_e_w_m_r',
data=b'None ' + exc_info.encode())
async def sync_worker_files(self, task_id: str,
received_file: asyncio.Event, logger):
"""Wait until extra valid files are received from the worker and create a child process for them.
Parameters
----------
task_id : str
Task ID to which the file was sent.
received_file : asyncio.Event
Asyncio event that is holding a lock while the files are not received.
logger : Logger object
Logger to use (can't use self since one of the task loggers will be used).
"""
logger.debug("Waiting to receive zip file from worker.")
await asyncio.wait_for(received_file.wait(),
timeout=self.cluster_items['intervals']
['communication']['timeout_receiving_file'])
# Full path where the zip sent by the worker is located.
received_filename = self.sync_tasks[task_id].filename
if isinstance(received_filename, Exception):
raise received_filename
logger.debug(
f"Received extra-valid file from worker: '{received_filename}'")
# Path to metadata file (files_metadata.json) and to zipdir (directory with decompressed files).
files_metadata, decompressed_files_path = await wazuh.core.cluster.cluster.async_decompress_files(
received_filename)
logger.debug(
f"Received {len(files_metadata)} extra-valid files to check.")
# Create a child process to run the task.
try:
result = await cluster.run_in_pool(
self.loop, self.server.task_pool,
self.process_files_from_worker, files_metadata,
decompressed_files_path, self.cluster_items, self.name,
self.cluster_items['intervals']['master']
['timeout_extra_valid'])
except Exception as e:
raise exception.WazuhClusterError(3038, extra_message=str(e))
finally:
shutil.rmtree(decompressed_files_path)
# Log any possible error found in the process.
self.integrity_sync_status['total_extra_valid'] = result[
'total_updated']
if result['errors_per_folder']:
logger.error(
f"Errors updating worker files: {dict(result['errors_per_folder'])}"
)
for error in result['generic_errors']:
logger.error(error)
async def send_request(self, command: bytes, data: bytes) -> bytes:
"""Send a request to peer and wait for the response to be received and processed.
Parameters
----------
command : bytes
Command to send.
data : bytes
Data to send.
Returns
-------
response_data : bytes
Response from peer.
"""
if len(data) > self.request_chunk:
raise exception.WazuhClusterError(3033)
response = Response()
msg_counter = self.next_counter()
self.box[msg_counter] = response
try:
self.push(self.msg_build(command, msg_counter, data))
except MemoryError:
self.request_chunk //= 2
raise exception.WazuhClusterError(3026)
except Exception as e:
raise exception.WazuhClusterError(3018, extra_message=str(e))
try:
# A lock is hold until response.write() is called inside data_received() method.
response_data = await asyncio.wait_for(
response.read(),
timeout=self.cluster_items['intervals']['communication']
['timeout_cluster_request'])
del self.box[msg_counter]
except asyncio.TimeoutError:
self.box[msg_counter] = None
return b'Error sending request: timeout expired.'
return response_data
def process_error_from_peer(self, data: bytes) -> bytes:
"""
Handles errors in requests
:param data: error message from peer
:return: Nothing
"""
try:
exc = json.loads(data.decode(), object_hook=as_wazuh_object)
except json.JSONDecodeError as e:
exc = exception.WazuhClusterError(3000, extra_message=data.decode())
return exc
def end_receiving_file(self,
task_and_file_names: str) -> Tuple[bytes, bytes]:
task_name, filename = task_and_file_names.split(' ', 1)
if task_name not in self.sync_tasks:
# Remove filename if task_name does not exists, before raising exception
if os.path.exists(os.path.join(common.ossec_path, filename)):
try:
os.remove(os.path.join(common.ossec_path, filename))
except Exception as e:
self.get_logger(self.logger_tag).error(
f"Attempt to delete file {os.path.join(common.ossec_path, filename)} failed: {e}"
)
raise exception.WazuhClusterError(3027, extra_message=task_name)
self.sync_tasks[task_name].filename = os.path.join(
common.ossec_path, filename)
self.sync_tasks[task_name].received_information.set()
return b'ok', b'File correctly received'
