async def sync_integrity(self, task_name: str, received_file: asyncio.Event):
logger = self.task_loggers['Integrity']
self.sync_integrity_status['date_start_master'] = str(datetime.now())
logger.info("Waiting to receive zip file from worker")
await asyncio.wait_for(received_file.wait(),
timeout=self.cluster_items['intervals']['communication']['timeout_receiving_file'])
received_filename = self.sync_tasks[task_name].filename
if received_filename == 'Error':
logger.info("Stopping synchronization process: worker files weren't correctly received.")
return
logger.debug("Received file from worker: '{}'".format(received_filename))
files_checksums, decompressed_files_path = cluster.decompress_files(received_filename)
logger.info("Analyzing worker integrity: Received {} files to check.".format(len(files_checksums)))
# classify files in shared, missing, extra and extra valid.
worker_files_ko, counts = cluster.compare_files(self.server.integrity_control, files_checksums, self.name)
# health check
self.sync_integrity_status['total_files'] = counts
shutil.rmtree(decompressed_files_path)
if not functools.reduce(operator.add, map(len, worker_files_ko.values())):
logger.info("Analyzing worker integrity: Files checked. There are no KO files.")
result = await self.send_request(command=b'sync_m_c_ok', data=b'')
else:
logger.info("Analyzing worker integrity: Files checked. There are KO files.")
# Compress data: master files (only KO shared and missing)
logger.debug("Analyzing worker integrity: Files checked. Compressing KO files.")
master_files_paths = worker_files_ko['shared'].keys() | worker_files_ko['missing'].keys()
compressed_data = cluster.compress_files(self.name, master_files_paths, worker_files_ko)
logger.info("Analyzing worker integrity: Files checked. KO files compressed.")
task_name = await self.send_request(command=b'sync_m_c', data=b'')
if task_name.startswith(b'Error'):
logger.error(task_name.decode())
return task_name
result = await self.send_file(compressed_data)
os.unlink(compressed_data)
if result.startswith(b'Error'):
self.logger.error("Error sending files information: {}".format(result.decode()))
result = await self.send_request(command=b'sync_m_c_e', data=task_name + b' ' + b'Error')
else:
result = await self.send_request(command=b'sync_m_c_e',
data=task_name + b' ' + compressed_data.replace(common.ossec_path, '').encode())
if result.startswith(b'Error'):
self.logger.error(result.decode())
self.sync_integrity_status['date_end_master'] = str(datetime.now())
self.sync_integrity_free = True
logger.info("Finished integrity synchronization.")
return result
async def sync_worker_files(self, task_name: str, received_file: asyncio.Event, logger):
logger.info("Waiting to receive zip file from worker")
await asyncio.wait_for(received_file.wait(),
timeout=self.cluster_items['intervals']['communication']['timeout_receiving_file'])
received_filename = self.sync_tasks[task_name].filename
if received_filename == 'Error':
logger.info("Stopping synchronization process: worker files weren't correctly received.")
return
logger.debug("Received file from worker: '{}'".format(received_filename))
files_checksums, decompressed_files_path = cluster.decompress_files(received_filename)
logger.info("Analyzing worker files: Received {} files to check.".format(len(files_checksums)))
self.process_files_from_worker(files_checksums, decompressed_files_path, logger)
def process_files_from_client(self, client_name, data_received, cluster_control_key, cluster_control_subkey, tag=None):
sync_result = False
# Save info for healthcheck
self.manager.set_client_status(client_id=self.name, key=cluster_control_key, subkey="date_start_master", status=datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-4])
self.manager.set_client_status(client_id=self.name, key=cluster_control_key, subkey="date_end_master", status="In progress")
self.manager.set_client_status(client_id=self.name, key=cluster_control_key, subkey=cluster_control_subkey, status="In progress")
# ---
if not tag:
tag = "[Master] [process_files_from_client]"
# Extract received data
logger.info("{0}: Analyzing received files: Start.".format(tag))
try:
json_file, zip_dir_path = decompress_files(data_received)
except Exception as e:
logger.error("{0}: Error decompressing data: {1}.".format(tag, str(e)))
raise e
if json_file:
client_files_json = json_file['client_files']
else:
raise Exception("cluster_control.json not included in received zip file")
logger.info("{0}: Analyzing received files: End.".format(tag))
logger.info("{0}: Updating master files: Start.".format(tag))
# Update files
self._update_client_files_in_master(client_files_json, zip_dir_path, client_name,
cluster_control_key, cluster_control_subkey,
tag)
# Remove tmp directory created when zip file was received
shutil.rmtree(zip_dir_path)
logger.info("{0}: Updating master files: End.".format(tag))
sync_result = True
# Save info for healthcheck
self.manager.set_client_status(client_id=self.name, key=cluster_control_key, subkey="date_end_master", status=datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-4])
return sync_result
def process_files_from_master(self, data_received, tag=None):
if not tag:
tag = "[Worker] [process_files_from_master]"
logger.info("{0}: Analyzing received files: Start.".format(tag))
try:
ko_files, zip_path = decompress_files(data_received)
except Exception as e:
logger.error("{}: Error decompressing files from master: {}".format(tag, str(e)))
raise e
if ko_files:
logger.info("{0}: Analyzing received files: Missing: {1}. Shared: {2}. Extra: {3}. ExtraValid: {4}".format(tag, len(ko_files['missing']), len(ko_files['shared']), len(ko_files['extra']), len(ko_files['extra_valid'])))
logger.debug2("{0}: Received cluster_control.json: {1}".format(tag, ko_files))
else:
raise Exception("cluster_control.json not included in received zip file.")
logger.info("{0}: Analyzing received files: End.".format(tag))
# Update files
if ko_files['extra_valid']:
logger.info("{0}: Master requires some worker files. Sending.".format(tag))
if not "SyncExtraValidFilesThread" in set(map(lambda x: type(x).__name__, threading.enumerate())):
req_files_thread = SyncExtraValidFilesThread(self, self.stopper, ko_files['extra_valid'])
req_files_thread.start()
else:
logger.warning("{}: The last master's file request is in progress. Rejecting this request.".format(tag))
if not ko_files['shared'] and not ko_files['missing'] and not ko_files['extra']:
logger.info("{0}: Worker meets integrity checks. No actions.".format(tag))
sync_result = True
else:
logger.info("{0}: Worker does not meet integrity checks. Actions required.".format(tag))
logger.info("{0}: Updating files: Start.".format(tag))
sync_result = self._update_master_files_in_worker(ko_files, zip_path, tag)
logger.info("{0}: Updating files: End.".format(tag))
# remove temporal zip file directory
shutil.rmtree(zip_path)
return sync_result
async def process_files_from_master(self, name: str,
file_received: asyncio.Event):
await asyncio.wait_for(file_received.wait(),
timeout=self.cluster_items['intervals']
['communication']['timeout_receiving_file'])
received_filename = self.sync_tasks[name].filename
if received_filename == 'Error':
self.logger.info(
"Stopping synchronization process: worker files weren't correctly received."
)
return
try:
logger = self.task_loggers['Integrity']
logger.info("Analyzing received files: Start.")
ko_files, zip_path = cluster.decompress_files(received_filename)
logger.info(
"Analyzing received files: Missing: {}. Shared: {}. Extra: {}. ExtraValid: {}"
.format(len(ko_files['missing']), len(ko_files['shared']),
len(ko_files['extra']), len(ko_files['extra_valid'])))
# Update files
if ko_files['extra_valid']:
logger.info("Master requires some worker files.")
asyncio.create_task(
self.sync_extra_valid(ko_files['extra_valid']))
if not ko_files['shared'] and not ko_files[
'missing'] and not ko_files['extra']:
logger.info("Worker meets integrity checks. No actions.")
else:
logger.info(
"Worker does not meet integrity checks. Actions required.")
logger.info("Updating files: Start.")
self.update_master_files_in_worker(ko_files, zip_path)
logger.info("Updating files: End.")
finally:
shutil.rmtree(zip_path)
def process_integrity_from_worker(self,
worker_name,
data_received,
cluster_control_key,
cluster_control_subkey,
tag=None):
if not tag:
tag = "[Master] [process_integrity_from_worker]"
# Extract received data
logger.info("{0}: Analyzing worker integrity: Start.".format(tag))
try:
json_file, zip_dir_path = decompress_files(data_received)
except Exception as e:
logger.error("{0}: Error decompressing data: {1}".format(
tag, str(e)))
raise e
if json_file:
master_files_from_worker = json_file['master_files']
else:
raise Exception(
"cluster_control.json not included in received zip file")
logger.info(
"{0}: Analyzing worker integrity: Received {1} files to check.".
format(tag, len(master_files_from_worker)))
logger.info("{0}: Analyzing worker integrity: Checking files.".format(
tag, len(master_files_from_worker)))
# Get master files
master_files = self.server.get_integrity_control()
# Compare
worker_files_ko = compare_files(master_files, master_files_from_worker)
agent_groups_to_merge = {
key: fnmatch.filter(values.keys(), '*/agent-groups/*')
for key, values in worker_files_ko.items()
}
merged_files = {
key: merge_agent_info(merge_type="agent-groups",
files=values,
file_type="-" + key,
time_limit_seconds=0)
for key, values in agent_groups_to_merge.items()
}
for ko, merged in zip(worker_files_ko.items(),
agent_groups_to_merge.items()):
ko_type, ko_files = ko
if ko_type == "extra" or "extra_valid":
continue
_, merged_filenames = merged
for m in merged_filenames:
del ko_files[m]
n_files, merged_file = merged_files[ko_type]
if n_files > 0:
ko_files[merged_file] = {
'cluster_item_key': '/queue/agent-groups/',
'merged': True
}
# Save info for healthcheck
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="missing",
status=len(worker_files_ko['missing']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="shared",
status=len(worker_files_ko['shared']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="extra",
status=len(worker_files_ko['extra']))
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
subsubkey="extra_valid",
status=len(
worker_files_ko['extra_valid']))
# ---
# Remove tmp directory created when zip file was received
shutil.rmtree(zip_dir_path)
# Step 3: KO files
if len(list(filter(lambda x: x == {},
worker_files_ko.values()))) == len(worker_files_ko):
logger.info(
"{0}: Analyzing worker integrity: Files checked. There are no KO files."
.format(tag))
ko_files = False
data_for_worker = None
else:
logger.info(
"{0}: Analyzing worker integrity: Files checked. There are KO files."
.format(tag))
# Compress data: master files (only KO shared and missing)
logger.debug(
"{0} Analyzing worker integrity: Files checked. Compressing KO files."
.format(tag))
master_files_paths = [item for item in worker_files_ko['shared']]
master_files_paths.extend(
[item for item in worker_files_ko['missing']])
compressed_data = compress_files(worker_name, master_files_paths,
worker_files_ko)
logger.debug(
"{0} Analyzing worker integrity: Files checked. KO files compressed."
.format(tag))
ko_files = True
data_for_worker = compressed_data
logger.info("{0}: Analyzing worker integrity: End.".format(tag))
return ko_files, data_for_worker
