def send_extra_valid_files_to_master(self, files, reason=None, tag=None):
if not tag:
tag = "[Worker] [ReqFiles ]"
logger.info("{}: Start. Reason: '{}'.".format(tag, reason))
master_node = self.config['nodes'][0] # Now, we only have 1 node: the master
logger.info("{0}: Master found: {1}.".format(tag, master_node))
agent_groups_to_merge = set(fnmatch.filter(files.keys(), '*/agent-groups/*'))
if agent_groups_to_merge:
n_files, merged_file = merge_agent_info(merge_type='agent-groups',
files=agent_groups_to_merge,
time_limit_seconds=0)
for ag in agent_groups_to_merge:
del files[ag]
if n_files:
files.update({merged_file: {'merged': True,
'merge_name': merged_file,
'merge_type': 'agent-groups',
'cluster_item_key': '/queue/agent-groups/'}})
compressed_data_path = compress_files(self.name, files, {'worker_files': files})
return compressed_data_path
async def sync_extra_valid(self, extra_valid: Dict):
extra_valid_logger = self.task_loggers["Extra valid"]
try:
before = time.time()
self.logger.debug("Starting to send extra valid files")
# TODO: Add support for more extra valid file types if ever added
n_files, merged_file = cluster.merge_agent_info(
merge_type='agent-groups',
files=extra_valid.keys(),
time_limit_seconds=0,
node_name=self.name)
if n_files:
files_to_sync = {
merged_file: {
'merged': True,
'merge_type': 'agent-groups',
'merge_name': merged_file,
'cluster_item_key': '/queue/agent-groups/'
}
}
my_worker = SyncWorker(cmd=b'sync_e_w_m',
files_to_sync=files_to_sync,
checksums=files_to_sync,
logger=extra_valid_logger,
worker=self)
await my_worker.sync()
after = time.time()
self.logger.debug2(
"Time synchronizing extra valid files: {} s".format(after -
before))
except Exception as e:
extra_valid_logger.error(
"Error synchronizing extra valid files: {}".format(e))
res = await self.send_request(command=b'sync_e_w_m_r',
data=str(e).encode())
def test_merge_agent_info(stat_mock, listdir_mock):
"""
Tests merge agent info function
"""
stat_mock.return_value.st_mtime = time.time()
stat_mock.return_value.st_size = len(agent_info)
with patch('builtins.open', mock_open(read_data=agent_info)) as m:
cluster.merge_agent_info('agent-info', 'worker1')
m.assert_any_call('/var/ossec/queue/cluster/worker1/agent-info.merged',
'wb')
m.assert_any_call('/var/ossec/queue/agent-info/agent1-any', 'rb')
m.assert_any_call('/var/ossec/queue/agent-info/agent2-any', 'rb')
handle = m()
expected = f'{len(agent_info)} agent1-any {datetime.utcfromtimestamp(stat_mock.return_value.st_mtime)}\n'.encode(
) + agent_info
handle.write.assert_any_call(expected)
def process_integrity_from_worker(self,
worker_name,
data_received,
cluster_control_key,
cluster_control_subkey,
tag=None):
if not tag:
tag = "[Master] [process_integrity_from_worker]"
# Extract received data
logger.info("{0}: Analyzing worker integrity: Start.".format(tag))
try:
json_file, zip_dir_path = decompress_files(data_received)
except Exception as e:
logger.error("{0}: Error decompressing data: {1}".format(
tag, str(e)))
raise e
if json_file:
master_files_from_worker = json_file['master_files']
else:
raise Exception(
"cluster_control.json not included in received zip file")
logger.info(
"{0}: Analyzing worker integrity: Received {1} files to check.".
format(tag, len(master_files_from_worker)))
logger.info("{0}: Analyzing worker integrity: Checking files.".format(
tag, len(master_files_from_worker)))
# Get master files
master_files = self.server.get_integrity_control()
# Compare
worker_files_ko = compare_files(master_files, master_files_from_worker)
agent_groups_to_merge = {
key: fnmatch.filter(values.keys(), '*/agent-groups/*')
for key, values in worker_files_ko.items()
}
merged_files = {
key: merge_agent_info(merge_type="agent-groups",
files=values,
file_type="-" + key,
time_limit_seconds=0)
for key, values in agent_groups_to_merge.items()
}
for ko, merged in zip(worker_files_ko.items(),
agent_groups_to_merge.items()):
ko_type, ko_files = ko
if ko_type == "extra" or "extra_valid":
continue
_, merged_filenames = merged
for m in merged_filenames:
del ko_files[m]
n_files, merged_file = merged_files[ko_type]
if n_files > 0:
ko_files[merged_file] = {
'cluster_item_key': '/queue/agent-groups/',
'merged': True
}
# Save info for healthcheck
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="missing",
status=len(worker_files_ko['missing']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="shared",
status=len(worker_files_ko['shared']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="extra",
status=len(worker_files_ko['extra']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="extra_valid",
status=len(
worker_files_ko['extra_valid']))
# ---
# Remove tmp directory created when zip file was received
shutil.rmtree(zip_dir_path)
# Step 3: KO files
if len(list(filter(lambda x: x == {},
worker_files_ko.values()))) == len(worker_files_ko):
logger.info(
"{0}: Analyzing worker integrity: Files checked. There are no KO files."
.format(tag))
ko_files = False
data_for_worker = None
else:
logger.info(
"{0}: Analyzing worker integrity: Files checked. There are KO files."
.format(tag))
# Compress data: master files (only KO shared and missing)
logger.debug(
"{0} Analyzing worker integrity: Files checked. Compressing KO files."
.format(tag))
master_files_paths = [item for item in worker_files_ko['shared']]
master_files_paths.extend(
[item for item in worker_files_ko['missing']])
compressed_data = compress_files(worker_name, master_files_paths,
worker_files_ko)
logger.debug(
"{0} Analyzing worker integrity: Files checked. KO files compressed."
.format(tag))
ko_files = True
data_for_worker = compressed_data
logger.info("{0}: Analyzing worker integrity: End.".format(tag))
return ko_files, data_for_worker
