def test_unmerge_agent_info(stat_mock, agent_info, exception):
stat_mock.return_value.st_size = len(agent_info)
with patch('builtins.open', mock_open(read_data=agent_info)) as m:
agent_infos = list(
cluster.unmerge_agent_info('agent-info', '/random/path',
'agent-info.merged'))
assert len(agent_infos) == (1 if exception is None else 0)
def overwrite_or_create_files(filename, data):
full_filename_path = common.ossec_path + filename
if os.path.basename(filename) == 'client.keys':
self._check_removed_agents("{}{}".format(zip_path, filename),
logger)
if data['merged']: # worker nodes can only receive agent-groups files
if data['merge-type'] == 'agent-info':
logger.warning("Agent status received in a worker node")
raise WazuhException(3011)
for name, content, _ in cluster.unmerge_agent_info(
'agent-groups', zip_path, filename):
full_unmerged_name = common.ossec_path + name
tmp_unmerged_path = full_unmerged_name + '.tmp'
with open(tmp_unmerged_path, 'wb') as f:
f.write(content)
os.chown(tmp_unmerged_path, common.ossec_uid,
common.ossec_gid)
os.rename(tmp_unmerged_path, full_unmerged_name)
else:
if not os.path.exists(os.path.dirname(full_filename_path)):
utils.mkdir_with_mode(os.path.dirname(full_filename_path))
os.rename("{}{}".format(zip_path, filename),
full_filename_path)
os.chown(full_filename_path, common.ossec_uid,
common.ossec_gid)
os.chmod(
full_filename_path, self.cluster_items['files'][
data['cluster_item_key']]['permissions'])
def overwrite_or_create_files(filename: str, data: Dict):
"""
Updates a file coming from the master
:param filename: Filename to update
:param data: File metadata such as modification time, whether it's a merged file or not, etc.
:return: None
"""
full_filename_path = common.ossec_path + filename
if os.path.basename(filename) == 'client.keys':
self._check_removed_agents("{}{}".format(zip_path, filename), logger)
if data['merged']: # worker nodes can only receive agent-groups files
if data['merge-type'] == 'agent-info':
logger.warning("Agent status received in a worker node")
raise WazuhException(3011)
for name, content, _ in cluster.unmerge_agent_info('agent-groups', zip_path, filename):
full_unmerged_name = os.path.join(common.ossec_path, name)
tmp_unmerged_path = full_unmerged_name + '.tmp'
with open(tmp_unmerged_path, 'wb') as f:
f.write(content)
safe_move(tmp_unmerged_path, full_unmerged_name,
permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions'],
ownership=(common.ossec_uid(), common.ossec_gid())
)
else:
if not os.path.exists(os.path.dirname(full_filename_path)):
utils.mkdir_with_mode(os.path.dirname(full_filename_path))
safe_move("{}{}".format(zip_path, filename), full_filename_path,
permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions'],
ownership=(common.ossec_uid(), common.ossec_gid())
)
async def update_file(name: str, data: Dict):
"""
Updates a file from the worker. It checks the modification date to decide whether to update it or not.
If it's a merged file, it unmerges it.
:param name: Filename to update
:param data: File metadata
:return: None
"""
# Full path
full_path, error_updating_file, n_merged_files = common.ossec_path + name, False, 0
# Cluster items information: write mode and permissions
lock_full_path = "{}/queue/cluster/lockdir/{}.lock".format(common.ossec_path, os.path.basename(full_path))
lock_file = open(lock_full_path, 'a+')
try:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
if os.path.basename(name) == 'client.keys':
self.logger.warning("Client.keys received in a master node")
raise WazuhException(3007)
if data['merged']:
is_agent_info = data['merge_type'] == 'agent-info'
if is_agent_info:
self.sync_agent_info_status['total_agent_info'] = len(agent_ids)
else:
self.sync_extra_valid_status['total_extra_valid'] = len(agent_ids)
for file_path, file_data, file_time in cluster.unmerge_agent_info(data['merge_type'],
decompressed_files_path,
data['merge_name']):
full_unmerged_name = os.path.join(common.ossec_path, file_path)
tmp_unmerged_path = os.path.join(common.ossec_path, 'queue/cluster', self.name, os.path.basename(file_path))
try:
if is_agent_info:
agent_name_re = re.match(r'(^.+)-(.+)$', os.path.basename(file_path))
agent_name = agent_name_re.group(1) if agent_name_re else os.path.basename(file_path)
if agent_name not in agent_names:
n_errors['warnings'][data['cluster_item_key']] = 1 \
if n_errors['warnings'].get(data['cluster_item_key']) is None \
else n_errors['warnings'][data['cluster_item_key']] + 1
self.logger.debug2("Received status of an non-existent agent '{}'".format(agent_name))
continue
else:
agent_id = os.path.basename(file_path)
if agent_id not in agent_ids:
n_errors['warnings'][data['cluster_item_key']] = 1 \
if n_errors['warnings'].get(data['cluster_item_key']) is None \
else n_errors['warnings'][data['cluster_item_key']] + 1
self.logger.debug2("Received group of an non-existent agent '{}'".format(agent_id))
continue
try:
mtime = datetime.strptime(file_time, '%Y-%m-%d %H:%M:%S.%f')
except ValueError:
mtime = datetime.strptime(file_time, '%Y-%m-%d %H:%M:%S')
if os.path.isfile(full_unmerged_name):
local_mtime = datetime.utcfromtimestamp(int(os.stat(full_unmerged_name).st_mtime))
# check if the date is older than the manager's date
if local_mtime > mtime:
logger.debug2("Receiving an old file ({})".format(file_path))
continue
with open(tmp_unmerged_path, 'wb') as f:
f.write(file_data)
mtime_epoch = timegm(mtime.timetuple())
utils.safe_move(tmp_unmerged_path, full_unmerged_name,
ownership=(common.ossec_uid(), common.ossec_gid()),
permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions'],
time=(mtime_epoch, mtime_epoch)
)
except Exception as e:
self.logger.error("Error updating agent group/status ({}): {}".format(tmp_unmerged_path, e))
if is_agent_info:
self.sync_agent_info_status['total_agent_info'] -= 1
else:
self.sync_extra_valid_status['total_extra_valid'] -= 1
n_errors['errors'][data['cluster_item_key']] = 1 \
if n_errors['errors'].get(data['cluster_item_key']) is None \
else n_errors['errors'][data['cluster_item_key']] + 1
await asyncio.sleep(0.0001)
else:
zip_path = "{}{}".format(decompressed_files_path, name)
utils.safe_move(zip_path, full_path,
ownership=(common.ossec_uid(), common.ossec_gid()),
permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions']
)
except WazuhException as e:
logger.debug2("Warning updating file '{}': {}".format(name, e))
error_tag = 'warnings'
error_updating_file = True
except Exception as e:
logger.debug2("Error updating file '{}': {}".format(name, e))
error_tag = 'errors'
error_updating_file = True
if error_updating_file:
n_errors[error_tag][data['cluster_item_key']] = 1 if not n_errors[error_tag].get(
data['cluster_item_key']) \
else n_errors[error_tag][data['cluster_item_key']] + 1
fcntl.lockf(lock_file, fcntl.LOCK_UN)
lock_file.close()
def _update_worker_files_in_master(self, json_file, zip_dir_path,
worker_name, cluster_control_key,
cluster_control_subkey, tag):
def update_file(n_errors,
name,
data,
file_time=None,
content=None,
agents=None):
# Full path
full_path = common.ossec_path + name
error_updating_file = False
# Cluster items information: write mode and umask
w_mode = cluster_items[data['cluster_item_key']]['write_mode']
umask = cluster_items[data['cluster_item_key']]['umask']
if content is None:
zip_path = "{}/{}".format(zip_dir_path, name)
with open(zip_path, 'rb') as f:
content = f.read()
lock_full_path = "{}/queue/cluster/lockdir/{}.lock".format(
common.ossec_path, os.path.basename(full_path))
lock_file = open(lock_full_path, 'a+')
try:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
_update_file(file_path=name,
new_content=content,
umask_int=umask,
mtime=file_time,
w_mode=w_mode,
tmp_dir=tmp_path,
whoami='master',
agents=agents)
except WazuhException as e:
logger.debug2("{}: Warning updating file '{}': {}".format(
tag, name, e))
error_tag = 'warnings'
error_updating_file = True
except Exception as e:
logger.debug2("{}: Error updating file '{}': {}".format(
tag, name, e))
error_tag = 'errors'
error_updating_file = True
if error_updating_file:
n_errors[error_tag][data['cluster_item_key']] = 1 if not n_errors[error_tag].get(data['cluster_item_key']) \
else n_errors[error_tag][data['cluster_item_key']] + 1
fcntl.lockf(lock_file, fcntl.LOCK_UN)
lock_file.close()
return n_errors, error_updating_file
# tmp path
tmp_path = "/queue/cluster/{}/tmp_files".format(worker_name)
cluster_items = get_cluster_items()['files']
n_merged_files = 0
n_errors = {'errors': {}, 'warnings': {}}
# create temporary directory for lock files
lock_directory = "{}/queue/cluster/lockdir".format(common.ossec_path)
if not os.path.exists(lock_directory):
mkdir_with_mode(lock_directory)
try:
agents = Agent.get_agents_overview(select={'fields': ['name']},
limit=None)['items']
agent_names = set(map(itemgetter('name'), agents))
agent_ids = set(map(itemgetter('id'), agents))
except Exception as e:
logger.debug2("{}: Error getting agent ids and names: {}".format(
tag, e))
agent_names, agent_ids = {}, {}
before = time.time()
try:
for filename, data in json_file.items():
if data['merged']:
for file_path, file_data, file_time in unmerge_agent_info(
data['merge_type'], zip_dir_path,
data['merge_name']):
n_errors, error_updating_file = update_file(
n_errors, file_path, data, file_time, file_data,
(agent_names, agent_ids))
if not error_updating_file:
n_merged_files += 1
if self.stopper.is_set():
break
else:
n_errors, _ = update_file(n_errors, filename, data)
except Exception as e:
logger.error("{}: Error updating worker files: '{}'.".format(
tag, e))
raise e
after = time.time()
logger.debug(
"{0}: Time updating worker files: {1:.2f}s. Total of updated worker files: {2}."
.format(tag, after - before, n_merged_files))
if sum(n_errors['errors'].values()) > 0:
logging.error("{}: Errors updating worker files: {}".format(
tag, ' | '.join([
'{}: {}'.format(key, value)
for key, value in n_errors['errors'].items()
])))
if sum(n_errors['warnings'].values()) > 0:
for key, value in n_errors['warnings'].items():
if key == '/queue/agent-info/':
logger.debug2(
"Received {} agent statuses for non-existent agents. Skipping."
.format(value))
elif key == '/queue/agent-groups/':
logger.debug2(
"Received {} group assignments for non-existent agents. Skipping."
.format(value))
# Save info for healthcheck
self.manager.set_worker_status(worker_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
status=n_merged_files)
def _update_master_files_in_worker(self, wrong_files, zip_path_dir, tag=None):
def overwrite_or_create_files(filename, data, content=None):
# Cluster items information: write mode and umask
cluster_item_key = data['cluster_item_key']
w_mode = cluster_items[cluster_item_key]['write_mode']
umask = cluster_items[cluster_item_key]['umask']
if content is None:
# Full path
file_path = common.ossec_path + filename
zip_path = "{}/{}".format(zip_path_dir, filename)
# File content and time
with open(zip_path, 'r') as f:
file_data = f.read()
else:
file_data = content
tmp_path='/queue/cluster/tmp_files'
_update_file(file_path=filename, new_content=file_data,
umask_int=umask, w_mode=w_mode, tmp_dir=tmp_path, whoami='worker')
if not tag:
tag = "[Worker] [Sync process]"
cluster_items = get_cluster_items()['files']
before = time.time()
error_shared_files = 0
if wrong_files['shared']:
logger.debug("{0}: Received {1} wrong files to fix from master. Action: Overwrite files.".format(tag, len(wrong_files['shared'])))
for file_to_overwrite, data in wrong_files['shared'].items():
try:
logger.debug2("{0}: Overwrite file: '{1}'".format(tag, file_to_overwrite))
if data['merged']:
for name, content, _ in unmerge_agent_info('agent-groups', zip_path_dir, file_to_overwrite):
overwrite_or_create_files(name, data, content)
if self.stopper.is_set():
break
else:
overwrite_or_create_files(file_to_overwrite, data)
if self.stopper.is_set():
break
except Exception as e:
error_shared_files += 1
logger.debug2("{}: Error overwriting file '{}': {}".format(tag, file_to_overwrite, str(e)))
continue
error_missing_files = 0
if wrong_files['missing']:
logger.debug("{0}: Received {1} missing files from master. Action: Create files.".format(tag, len(wrong_files['missing'])))
for file_to_create, data in wrong_files['missing'].items():
try:
logger.debug2("{0}: Create file: '{1}'".format(tag, file_to_create))
if data['merged']:
for name, content, _ in unmerge_agent_info('agent-groups', zip_path_dir, file_to_create):
overwrite_or_create_files(name, data, content)
if self.stopper.is_set():
break
else:
overwrite_or_create_files(file_to_create, data)
if self.stopper.is_set():
break
except Exception as e:
error_missing_files += 1
logger.debug2("{}: Error creating file '{}': {}".format(tag, file_to_create, str(e)))
continue
error_extra_files = 0
if wrong_files['extra']:
logger.debug("{0}: Received {1} extra files from master. Action: Remove files.".format(tag, len(wrong_files['extra'])))
for file_to_remove in wrong_files['extra']:
try:
logger.debug2("{0}: Remove file: '{1}'".format(tag, file_to_remove))
file_path = common.ossec_path + file_to_remove
try:
os.remove(file_path)
except OSError as e:
if e.errno == errno.ENOENT and '/queue/agent-groups/' in file_path:
logger.debug2("{}: File {} doesn't exist.".format(tag, file_to_remove))
continue
else:
raise e
except Exception as e:
error_extra_files += 1
logger.debug2("{}: Error removing file '{}': {}".format(tag, file_to_remove, str(e)))
continue
if self.stopper.is_set():
break
directories_to_check = {os.path.dirname(f): cluster_items[data\
['cluster_item_key']]['remove_subdirs_if_empty']
for f, data in wrong_files['extra'].items()}
for directory in map(itemgetter(0), filter(lambda x: x[1], directories_to_check.items())):
try:
full_path = common.ossec_path + directory
dir_files = set(os.listdir(full_path))
if not dir_files or dir_files.issubset(set(cluster_items['excluded_files'])):
shutil.rmtree(full_path)
except Exception as e:
error_extra_files += 1
logger.debug2("{}: Error removing directory '{}': {}".format(tag, directory, str(e)))
continue
if self.stopper.is_set():
break
if error_extra_files or error_shared_files or error_missing_files:
logger.error("{}: Found errors: {} overwriting, {} creating and {} removing".format(tag,
error_shared_files, error_missing_files, error_extra_files))
after = time.time()
logger.debug2("{}: Time updating integrity from master: {}s".format(tag, after - before))
return True
def _update_client_files_in_master(self, json_file, files_to_update_json,
zip_dir_path, client_name,
cluster_control_key,
cluster_control_subkey, tag):
def update_file(n_errors,
name,
data,
file_time=None,
content=None,
agents=None):
# Full path
full_path = common.ossec_path + name
# Cluster items information: write mode and umask
w_mode = cluster_items[data['cluster_item_key']]['write_mode']
umask = int(cluster_items[data['cluster_item_key']]['umask'],
base=0)
if content is None:
zip_path = "{}/{}".format(zip_dir_path, name)
with open(zip_path, 'rb') as f:
content = f.read()
lock_full_path = "{}/queue/cluster/lockdir/{}.lock".format(
common.ossec_path, os.path.basename(full_path))
lock_file = open(lock_full_path, 'a+')
try:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
_update_file(file_path=name,
new_content=content,
umask_int=umask,
mtime=file_time,
w_mode=w_mode,
tmp_dir=tmp_path,
whoami='master',
agents=agents)
except Exception as e:
logger.debug2("{}: Error updating file '{}': {}".format(
tag, name, e))
n_errors[data['cluster_item_key']] = 1 if not n_errors.get(data['cluster_item_key']) \
else n_errors[data['cluster_item_key']] + 1
fcntl.lockf(lock_file, fcntl.LOCK_UN)
lock_file.close()
return n_errors
# tmp path
tmp_path = "/queue/cluster/{}/tmp_files".format(client_name)
cluster_items = get_cluster_items()['files']
n_agentsinfo = 0
n_agentgroups = 0
n_errors = {}
# create temporary directory for lock files
lock_directory = "{}/queue/cluster/lockdir".format(common.ossec_path)
if not os.path.exists(lock_directory):
mkdir_with_mode(lock_directory)
try:
agents = Agent.get_agents_overview(select={'fields': ['name']},
limit=None)['items']
agent_names = set(map(itemgetter('name'), agents))
agent_ids = set(map(itemgetter('id'), agents))
agents = None
except Exception as e:
logger.debug2("{}: Error getting agent ids and names: {}".format(
tag, e))
agent_names, agent_ids = {}, {}
before = time.time()
try:
for filename, data in json_file.items():
if data['merged']:
for file_path, file_data, file_time in unmerge_agent_info(
data['merge_type'], zip_dir_path,
data['merge_name']):
n_errors = update_file(n_errors, file_path, data,
file_time, file_data,
(agent_names, agent_ids))
if data['merge_type'] == 'agent-info':
n_agentsinfo += 1
else:
n_agentgroups += 1
if self.stopper.is_set():
break
else:
n_errors = update_file(n_errors, filename, data)
except Exception as e:
logger.error("{}: Error updating client files: '{}'.".format(
tag, e))
raise e
after = time.time()
logger.debug(
"{0}: Time updating client files: {1:.2f}s. Agents-info updated total: {2}. Agent-groups updated total: {3}."
.format(tag, after - before, n_agentsinfo, n_agentgroups))
if sum(n_errors.values()) > 0:
logging.error("{}: Errors updating client files: {}".format(
tag, ' | '.join([
'{}: {}'.format(key, value)
for key, value in n_errors.items()
])))
# Save info for healthcheck
status_number = n_agentsinfo if cluster_control_key == 'last_sync_agentinfo' else n_agentgroups
self.manager.set_client_status(client_id=self.name,
key=cluster_control_key,
subkey=cluster_control_subkey,
status=status_number)
def update_file(name, data):
# Full path
full_path, error_updating_file, n_merged_files = common.ossec_path + name, False, 0
# Cluster items information: write mode and permissions
lock_full_path = "{}/queue/cluster/lockdir/{}.lock".format(
common.ossec_path, os.path.basename(full_path))
lock_file = open(lock_full_path, 'a+')
try:
fcntl.lockf(lock_file, fcntl.LOCK_EX)
if os.path.basename(name) == 'client.keys':
self.logger.warning(
"Client.keys received in a master node")
raise WazuhException(3007)
if data['merged']:
is_agent_info = data['merge_type'] == 'agent-info'
if is_agent_info:
self.sync_agent_info_status['total_agent_info'] = len(
agent_ids)
else:
self.sync_extra_valid_status[
'total_extra_valid'] = len(agent_ids)
for file_path, file_data, file_time in cluster.unmerge_agent_info(
data['merge_type'], decompressed_files_path,
data['merge_name']):
try:
full_unmerged_name = common.ossec_path + file_path
tmp_unmerged_path = full_unmerged_name + '.tmp'
if is_agent_info:
agent_name_re = re.match(
r'(^.+)-(.+)$',
os.path.basename(file_path))
agent_name = agent_name_re.group(
1) if agent_name_re else os.path.basename(
file_path)
if agent_name not in agent_names:
n_errors['warnings'][data['cluster_item_key']] = 1 \
if n_errors['warnings'].get(data['cluster_item_key']) is None \
else n_errors['warnings'][data['cluster_item_key']] + 1
self.logger.debug2(
"Received status of an non-existent agent '{}'"
.format(agent_name))
continue
else:
agent_id = os.path.basename(file_path)
if agent_id not in agent_ids:
n_errors['warnings'][data['cluster_item_key']] = 1 \
if n_errors['warnings'].get(data['cluster_item_key']) is None \
else n_errors['warnings'][data['cluster_item_key']] + 1
self.logger.debug2(
"Received group of an non-existent agent '{}'"
.format(agent_id))
continue
try:
mtime = datetime.strptime(
file_time, '%Y-%m-%d %H:%M:%S.%f')
except ValueError:
mtime = datetime.strptime(
file_time, '%Y-%m-%d %H:%M:%S')
if os.path.isfile(full_unmerged_name):
local_mtime = datetime.utcfromtimestamp(
int(os.stat(full_unmerged_name).st_mtime))
# check if the date is older than the manager's date
if local_mtime > mtime:
logger.debug2(
"Receiving an old file ({})".format(
file_path))
return
with open(tmp_unmerged_path, 'wb') as f:
f.write(file_data)
mtime_epoch = timegm(mtime.timetuple())
os.utime(
tmp_unmerged_path,
(mtime_epoch, mtime_epoch)) # (atime, mtime)
os.chown(tmp_unmerged_path, common.ossec_uid,
common.ossec_gid)
os.chmod(
tmp_unmerged_path, self.cluster_items['files'][
data['cluster_item_key']]['permissions'])
os.rename(tmp_unmerged_path, full_unmerged_name)
except Exception as e:
self.logger.debug2(
"Error updating agent group/status: {}".format(
e))
if is_agent_info:
self.sync_agent_info_status[
'total_agent_info'] -= 1
else:
self.sync_extra_valid_status[
'total_extra_valid'] -= 1
n_errors['errors'][data['cluster_item_key']] = 1 \
if n_errors['errors'].get(data['cluster_item_key']) is None \
else n_errors['errors'][data['cluster_item_key']] + 1
else:
zip_path = "{}{}".format(decompressed_files_path, name)
os.chown(zip_path, common.ossec_uid, common.ossec_gid)
os.chmod(
zip_path, self.cluster_items['files'][
data['cluster_item_key']]['permissions'])
os.rename(zip_path, full_path)
except WazuhException as e:
logger.debug2("Warning updating file '{}': {}".format(name, e))
error_tag = 'warnings'
error_updating_file = True
except Exception as e:
logger.debug2("Error updating file '{}': {}".format(name, e))
error_tag = 'errors'
error_updating_file = True
if error_updating_file:
n_errors[error_tag][data['cluster_item_key']] = 1 if not n_errors[error_tag].get(
data['cluster_item_key']) \
else n_errors[error_tag][data['cluster_item_key']] + 1
fcntl.lockf(lock_file, fcntl.LOCK_UN)
lock_file.close()
